def __init__(self, h, n, input_dims=(3, 64, 64)):
super(dicriminator_model, self).__init__()
self.n = n
self.h = h
channel, width, height = input_dims
encoder = []
self.blocks = int(np.log2(width) - 2)
encoder.append(nn.Conv2d(n, n, kernel_size=3, padding=1))
prev_channel_size = n
for i in range(self.blocks):
channel_size = (i+1) * n
encoder.append(nn.Conv2d(prev_channel_size, channel_size, kernel_size=3, padding=1))
encoder.append(nn.ELU())
encoder.append(nn.Conv2d(channel_size, channel_size, kernel_size=3, padding=1))
encoder.append(nn.ELU())
if i < self.blocks - 1:
encoder.append(nn.Conv2d(channel_size, channel_size, kernel_size=3, padding=1))
encoder.append(nn.ELU())
prev_channel_size = channel_size
self.encoder_model = nn.Sequential(*encoder)
self.fc1 = nn.Linear(8 * 8 * self.blocks * n, h)
decoder = []
for i in range(self.blocks):
decoder.append(nn.Conv2d(n, n, kernel_size=3, padding=1))
decoder.append(nn.ELU())
decoder.append(nn.Conv2d(n, n, kernel_size=3, padding=1))
decoder.append(nn.ELU())
if i < self.blocks - 1:
decoder.append(nn.UpsamplingNearest2d(scale_factor=2))
decoder.append(nn.Conv2d(n, channel, kernel_size=3, padding=1))
decoder_model = nn.Sequential(*decoder)
self.fc2 = nn.Linear(h, 8 * 8 * n)
def forward(self, x):
x = self.encoder_model(x).view(x.size(0), -1)
x = self.fc1(x)
x = self.fc2(x).view(-1, self.n, 8, 8)
x = self.decoder_model(x)
return x
def __init__(self,
embedding=128,
nb_words=0,
frag_size=0,
channels=0,
kernel_size=0,
padding=0,
stride=0,
dilation=0):
super(Encoder, self).__init__()
self.channels = channels
self.kernel_size = kernel_size
self.padding = padding
self.stride = stride
self.dilation = dilation
self.embedding = embedding
self.nb_words = nb_words
self.frag_size = frag_size
self.channel = 300
self.len_encoding = 500
self.embedder_words = nn.Embedding(self.nb_words,
self.embedding,
max_norm=5.0,
norm_type=2.0)
#self.embedder_words.weight.requires_grad=False
self.lstm = nn.LSTM(self.embedding,
self.embedding,
num_layers=3,
dropout=0.5,
bidirectional=True)
self.sru1 = SRU(
self.embedding,
self.embedding,
num_layers=4, # number of stacking RNN layers
dropout=0.2, # dropout applied between RNN layers
rnn_dropout=
0.2, # variational dropout applied on linear transformation
use_tanh=1, # use tanh?
use_relu=0, # use ReLU?
#use_selu = 0, # use SeLU?
bidirectional=True # bidirectional RNN ?
)
self.sru2 = SRU(
self.embedding * 2 * 4,
self.embedding,
num_layers=4, # number of stacking RNN layers
dropout=0.2, # dropout applied between RNN layers
rnn_dropout=
0.2, # variational dropout applied on linear transformation
use_tanh=1, # use tanh?
use_relu=0, # use ReLU?
#use_selu = 0, # use SeLU?
bidirectional=True # bidirectional RNN ?
)
self.att1 = nn.Parameter(torch.randn(self.embedding * 2 * 4) / 100)
self.att2 = nn.Parameter(torch.randn(self.embedding * 2 * 4) / 100)
self.att3 = nn.Parameter(torch.randn(self.len_encoding) / 100)
#self.channel = 256
self.fc1 = nn.Linear(1502, 300)
self.fc2 = nn.Linear(300, 300)
self.fc3 = nn.Linear(300, 300)
self.fc4 = nn.Linear(1000, 500)
#self.fc2 = nn.Linear(128, 64)
#self.fc1 = nn.Linear(20,1)
#self.fc2 = nn.Linear(128, 64)
#self.fc3 = nn.Linear(self.embedding*2 ,self.embedding)
self.dropout1 = nn.Dropout(0.5)
self.dropout2 = nn.Dropout(0.2)
self.sigmoid = nn.Sigmoid()
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.elu = nn.ELU()
self.leaky = nn.LeakyReLU()
self.drop1 = nn.Dropout2d(0.5)
self.softmax = nn.Softmax(dim=0)
#for encoding
self.conv_1 = nn.Conv2d(1,
self.channel,
kernel_size=(3, self.embedding),
padding=(1, 0),
stride=2,
dilation=1,
groups=1)
output_size1 = ((self.frag_size + (2 * 0) - 1 * (3 - 1) - 1) / 2) + 1
output_size2 = ((self.embedding + (2 * 0) - 1 *
(self.embedding - 1) - 1) / 2) + 1
#print(output_size1,output_size2,'after conv1')
self.conv_2 = nn.Conv2d(self.channel,
self.channel,
kernel_size=(500, 1),
padding=(0, 0),
stride=2,
dilation=1,
groups=1)
output_size1 = ((output_size1 + (2 * 0) - 1 * (3 - 1) - 1) / 2) + 1
output_size2 = ((output_size2 + (2 * 0) - 1 * (1 - 1) - 1) / 2) + 1
#print(output_size1,output_size2,'after conv2')
self.conv_3 = nn.Conv2d(self.channel,
self.len_encoding,
kernel_size=(1, 1),
padding=(0, 0),
stride=2,
dilation=1,
groups=1)
kernel_value_reversed = self.dilation * (
self.kernel_size - 1) + 2 #look at website of pytorch
self.conv_4 = nn.ConvTranspose2d(100,
self.channel,
kernel_size=(10, 1),
padding=(0, 0),
stride=10,
dilation=1,
groups=1)
output_size1 = ((output_size1 - 1) * (self.stride) -
(2 * self.padding) + (kernel_value_reversed))
output_size2 = ((output_size2 - 1) * (self.stride) -
(2 * self.padding) + (kernel_value_reversed))
#print(output_size1,output_size2,'after deconvpierwszy')
self.conv_5 = nn.ConvTranspose2d(self.channel,
self.channel,
kernel_size=(10, 1),
padding=(0, 0),
stride=10,
dilation=1,
groups=1)
#print(output_size1, output_size2,
# output_size1*output_size2*(self.channels),self.embedding,'after maxpool2')
self.conv_6 = nn.ConvTranspose2d(self.channel,
1,
kernel_size=(10, self.embedding),
padding=(0, 0),
stride=10,
dilation=1,
groups=1)
output_size1 = ((output_size1 - 1) * (self.stride) -
(2 * self.padding) + (kernel_value_reversed))
output_size2 = ((output_size2 - 1) * (self.stride) -
(2 * self.padding) + (kernel_value_reversed))
#print(output_size1,output_size2,'after decconv1drugi')
output_size1 = ((output_size1 - 1) * (self.stride) -
(2 * self.padding) + (kernel_value_reversed))
output_size2 = ((output_size2 - 1) * (self.stride) -
(2 * self.padding) + (kernel_value_reversed))
#print(output_size1,output_size2,'after decconv1dtrzeci')
#print(output_size1,output_size2,'last deconv')
self.splitter = nn.Linear(1, 50)
self.merger = nn.Linear(50, 1)
self.final_layer_words = torch.nn.Linear(self.embedding, self.nb_words)
for m in self.modules():
if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2.0 / n))
def main():
args = cfg.parse_args()
torch.cuda.manual_seed(args.rand_seed)
#switch datasets and models
if args.dataset == 'cifar':
from data.cifar import inf_train_gen
itr = inf_train_gen(args.batch_size, flip=False)
netE = Res18_Quadratic(3,
args.n_chan,
32,
normalize=False,
AF=nn.ELU())
#netE = SE_Res18_Quadratic(3,args.n_chan,32,normalize=False,AF=Swish())
elif args.dataset == 'mnist':
from data.mnist_32 import inf_train_gen
itr = inf_train_gen(args.batch_size)
netE = Res12_Quadratic(1,
args.n_chan,
32,
normalize=False,
AF=nn.ELU())
elif args.dataset == 'fmnist':
#print(dataset+str(args.n_chan))
from data.fashion_mnist_32 import inf_train_gen
itr = inf_train_gen(args.batch_size)
netE = Res12_Quadratic(1,
args.n_chan,
32,
normalize=False,
AF=nn.ELU())
else:
NotImplementedError('{} unknown dataset'.format(args.dataset))
#setup gpu
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
netE = netE.to(device)
if args.n_gpus > 1:
netE = nn.DataParallel(netE)
#setup path
now = datetime.now()
timestamp = now.strftime('%Y_%m_%d_%H_%M_%S')
#pdb.set_trace()
print(str(args.cont))
#print(str(args.time))
if args.cont == True:
root = 'logs/' + args.log + '_' + args.time #compose string for loading
#load network
file_name = 'netE_' + str(args.net_indx) + '.pt'
netE.load_state_dict(torch.load(root + '/models/' + file_name))
else: # start new will create logging folder
root = 'logs/' + args.log + '_' + timestamp #add timestemp
#over write if folder already exist, not likely to happen as timestamp is used
if os.path.isdir(root):
shutil.rmtree(root)
os.makedirs(root)
os.makedirs(root + '/models')
os.makedirs(root + '/samples')
writer = SummaryWriter(root)
# setup optimizer and lr scheduler
params = {'lr': args.max_lr, 'betas': (0.9, 0.95)}
optimizerE = torch.optim.Adam(netE.parameters(), **params)
if args.lr_schedule == 'exp':
scheduler = torch.optim.lr_scheduler.StepLR(optimizerE,
int(args.n_iter / 6))
elif args.lr_schedule == 'cosine':
scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizerE,
args.n_iter,
eta_min=1e-6,
last_epoch=-1)
elif args.lr_schedule == 'const':
scheduler = torch.optim.lr_scheduler.StepLR(optimizerE,
int(args.n_iter))
#train
print_interval = 50
max_iter = args.n_iter + args.net_indx
batchSize = args.batch_size
sigma0 = 0.1
sigma02 = sigma0**2
if args.noise_distribution == 'exp':
sigmas_np = np.logspace(np.log10(args.min_noise),
np.log10(args.max_noise), batchSize)
elif args.noise_distribution == 'lin':
sigmas_np = np.linspace(args.min_noise, args.max_noise, batchSize)
sigmas = torch.Tensor(sigmas_np).view((batchSize, 1, 1, 1)).to(device)
start_time = time.time()
for i in range(args.net_indx, args.net_indx + args.n_iter):
x_real = itr.__next__().to(device)
x_noisy = x_real + sigmas * torch.randn_like(x_real)
x_noisy = x_noisy.requires_grad_()
E = netE(x_noisy).sum()
grad_x = torch.autograd.grad(E, x_noisy, create_graph=True)[0]
x_noisy.detach()
optimizerE.zero_grad()
LS_loss = (((
(x_real - x_noisy) / sigmas / sigma02 + grad_x / sigmas)**2) /
batchSize).sum()
LS_loss.backward()
optimizerE.step()
scheduler.step()
if (i + 1) % print_interval == 0:
time_spent = time.time() - start_time
start_time = time.time()
netE.eval()
E_real = netE(x_real).mean()
E_noise = netE(torch.rand_like(x_real)).mean()
netE.train()
print(
'Iteration {}/{} ({:.0f}%), E_real {:e}, E_noise {:e}, Normalized Loss {:e}, time {:4.1f}'
.format(i + 1, max_iter, 100 * ((i + 1) / max_iter),
E_real.item(), E_noise.item(),
(sigma02**2) * (LS_loss.item()), time_spent))
writer.add_scalar('E_real', E_real.item(), i + 1)
writer.add_scalar('E_noise', E_noise.item(), i + 1)
writer.add_scalar('loss', (sigma02**2) * LS_loss.item(), i + 1)
del E_real, E_noise, x_real, x_noisy
if (i + 1) % args.save_every == 0:
print("-" * 50)
file_name = args.file_name + str(i + 1) + '.pt'
torch.save(netE.state_dict(), root + '/models/' + file_name)
def forward(
self,
X_float: torch.FloatTensor,
X_id_list: torch.LongTensor = None,
X_id_list_idxs: torch.LongTensor = None,
) -> torch.FloatTensor:
""" Forward pass for generic feed-forward DNNs. Assumes activation names
are valid pytorch activation names.
"""
id_feature_col_idx = 0
# concat embedded feature to the float feature tensor
for feature_name in self.id_feature_order:
embedding_table_id = self.feature_specs[feature_name]["product_set_id"]
embedding_table = self.embeddings[
self.embeddings_idx_map[embedding_table_id]
]
col_start = X_id_list_idxs[0][id_feature_col_idx]
col_end = X_id_list_idxs[0][id_feature_col_idx + 1]
embeddings = embedding_table(X_id_list[:, col_start:col_end])
# https://datascience.stackexchange.com/questions/44635/does-sum-of-embeddings-make-sense
# average the embeddings, but first drop the padded 0 embeddings
# using a shape that maintains the tensor dimensions
valid_row_mask = (embeddings.sum(dim=(2), keepdim=True) != 0).float()
denom = valid_row_mask.sum(dim=(1, 2)).unsqueeze(dim=1)
emedding_sum = embeddings.sum(dim=1)
avg_embeddings = emedding_sum / denom
# set Nan values to zero (case where there is an empty id list)
avg_embeddings[avg_embeddings != avg_embeddings] = 0
# concatenate the float tensor and embedded tensor
X_float = torch.cat((X_float, avg_embeddings), dim=1)
id_feature_col_idx += 2
x = X_float
for i, activation in enumerate(self.activations):
if self.use_batch_norm:
x = self.batch_norm_ops[i](x)
x = self.layers[i](x)
if self.use_layer_norm and i < len(self.layer_norm_ops):
x = self.layer_norm_ops[i](x)
if activation == "linear":
pass
elif activation == "tanh":
x = torch.tanh(x)
else:
x = getattr(F, activation)(x)
if self.use_dropout and i < len(self.dropout_layers):
x = self.dropout_layers[i](x)
if self.is_mdn and i == (len(self.activations)-2):
sigma = self.mdn_layer[0](x)
m = nn.ELU()
sigma = m(sigma) + 1.0
if self.is_classification:
x = F.softmax(x, dim=1)
if self.is_mdn:
x = torch.cat([x, sigma], dim=0)
return x
def __init__(self, nhidden):
super().__init__()
self.nhidden = nhidden
self.elu = nn.ELU()
self.fc1 = nn.Linear(1, nhidden)
self.fc2 = nn.Linear(nhidden, nhidden)
def create_conv(in_channels,
out_channels,
kernel_size,
order,
num_groups,
padding=1):
"""
Create a list of modules with together constitute a single conv layer with non-linearity
and optional batchnorm/groupnorm.
Args:
in_channels (int): number of input channels
out_channels (int): number of output channels
order (string): order of things, e.g.
'cr' -> conv + ReLU
'crg' -> conv + ReLU + groupnorm
'cl' -> conv + LeakyReLU
'ce' -> conv + ELU
num_groups (int): number of groups for the GroupNorm
padding (int): add zero-padding to the input
Return:
list of tuple (name, module)
"""
assert 'c' in order, "Conv layer MUST be present"
assert order[
0] not in 'rle', 'Non-linearity cannot be the first operation in the layer'
modules = []
for i, char in enumerate(order):
if char == 'r':
modules.append(('ReLU', nn.ReLU(inplace=True)))
elif char == 'l':
modules.append(
('LeakyReLU', nn.LeakyReLU(negative_slope=0.1, inplace=True)))
elif char == 'e':
modules.append(('ELU', nn.ELU(inplace=True)))
elif char == 'c':
# add learnable bias only in the absence of gatchnorm/groupnorm
bias = not ('g' in order or 'b' in order)
modules.append(('conv',
conv3d(in_channels,
out_channels,
kernel_size,
bias,
padding=padding)))
elif char == 'g':
is_before_conv = i < order.index('c')
assert not is_before_conv, 'GroupNorm MUST go after the Conv3d'
# number of groups must be less or equal the number of channels
if out_channels < num_groups:
num_groups = out_channels
modules.append(('groupnorm',
nn.GroupNorm(num_groups=num_groups,
num_channels=out_channels)))
elif char == 'b':
is_before_conv = i < order.index('c')
if is_before_conv:
modules.append(('batchnorm', nn.BatchNorm3d(in_channels)))
else:
modules.append(('batchnorm', nn.BatchNorm3d(out_channels)))
else:
raise ValueError(
f"Unsupported layer type '{char}'. MUST be one of ['b', 'g', 'r', 'l', 'e', 'c']"
)
return modules
def __init__(self,
input_dim,
output_dim,
kernel_size,
stride,
padding=0,
conv_padding=0,
dilation=1,
weight_norm='none',
norm='none',
activation='relu',
pad_type='zero',
transpose=False):
super(Conv2dBlock, self).__init__()
self.use_bias = True
# initialize padding
if pad_type == 'reflect':
self.pad = nn.ReflectionPad2d(padding)
elif pad_type == 'replicate':
self.pad = nn.ReplicationPad2d(padding)
elif pad_type == 'zero':
self.pad = nn.ZeroPad2d(padding)
elif pad_type == 'none':
self.pad = None
else:
assert 0, "Unsupported padding type: {}".format(pad_type)
# initialize normalization
norm_dim = output_dim
if norm == 'bn':
self.norm = nn.BatchNorm2d(norm_dim)
elif norm == 'in':
self.norm = nn.InstanceNorm2d(norm_dim)
elif norm == 'none':
self.norm = None
else:
assert 0, "Unsupported normalization: {}".format(norm)
if weight_norm == 'sn':
self.weight_norm = spectral_norm_fn
elif weight_norm == 'wn':
self.weight_norm = weight_norm_fn
elif weight_norm == 'none':
self.weight_norm = None
else:
assert 0, "Unsupported normalization: {}".format(weight_norm)
# initialize activation
if activation == 'relu':
self.activation = nn.ReLU(inplace=True)
elif activation == 'elu':
self.activation = nn.ELU(inplace=True)
elif activation == 'lrelu':
self.activation = nn.LeakyReLU(0.2, inplace=True)
elif activation == 'prelu':
self.activation = nn.PReLU()
elif activation == 'selu':
self.activation = nn.SELU(inplace=True)
elif activation == 'tanh':
self.activation = nn.Tanh()
elif activation == 'none':
self.activation = None
else:
assert 0, "Unsupported activation: {}".format(activation)
# initialize convolution
if transpose:
self.conv = nn.ConvTranspose2d(input_dim,
output_dim,
kernel_size,
stride,
padding=conv_padding,
output_padding=conv_padding,
dilation=dilation,
bias=self.use_bias)
else:
self.conv = nn.Conv2d(input_dim,
output_dim,
kernel_size,
stride,
padding=conv_padding,
dilation=dilation,
bias=self.use_bias)
if self.weight_norm:
self.conv = self.weight_norm(self.conv)
def __init__(self, resolution_inp=256, resolution_op=256, channel=6):
super(AntiSpoofing, self).__init__()
self.resolution_inp = resolution_inp
self.resolution_op = resolution_op
self.channel = channel
# define layers
self.zeropad2d = nn.ZeroPad2d((1, 1, 1, 1))
self.elu = nn.ELU()
self.conv1 = nn.Conv2d(in_channels=6,
out_channels=64,
kernel_size=3,
stride=1,
bias=False)
self.bn1 = nn.BatchNorm2d(num_features=64)
self.conv2 = nn.Conv2d(in_channels=64,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn2 = nn.BatchNorm2d(num_features=128)
self.conv3 = nn.Conv2d(in_channels=128,
out_channels=196,
kernel_size=3,
stride=1,
bias=False)
self.bn3 = nn.BatchNorm2d(num_features=196)
self.conv4 = nn.Conv2d(in_channels=196,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn4 = nn.BatchNorm2d(num_features=128)
self.pool1 = nn.MaxPool2d(kernel_size=2, stride=2)
self.conv5 = nn.Conv2d(in_channels=128,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn5 = nn.BatchNorm2d(num_features=128)
self.conv6 = nn.Conv2d(in_channels=128,
out_channels=196,
kernel_size=3,
stride=1,
bias=False)
self.bn6 = nn.BatchNorm2d(num_features=196)
self.conv7 = nn.Conv2d(in_channels=196,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn7 = nn.BatchNorm2d(num_features=128)
self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)
self.conv8 = nn.Conv2d(in_channels=128,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn8 = nn.BatchNorm2d(num_features=128)
self.conv9 = nn.Conv2d(in_channels=128,
out_channels=196,
kernel_size=3,
stride=1,
bias=False)
self.bn9 = nn.BatchNorm2d(num_features=196)
self.conv10 = nn.Conv2d(in_channels=196,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn10 = nn.BatchNorm2d(num_features=128)
self.pool3 = nn.MaxPool2d(kernel_size=2, stride=2)
self.conv11 = nn.Conv2d(in_channels=384,
out_channels=128,
kernel_size=3,
stride=1,
bias=False)
self.bn11 = nn.BatchNorm2d(num_features=128)
self.conv12 = nn.Conv2d(in_channels=128,
out_channels=3,
kernel_size=3,
stride=1,
bias=False)
self.bn12 = nn.BatchNorm2d(num_features=3)
self.conv13 = nn.Conv2d(in_channels=3,
out_channels=1,
kernel_size=3,
stride=1,
bias=False)
self.bn13 = nn.BatchNorm2d(num_features=1)
def conv2d(m, n, k, act=True):
layers = [nn.Conv2d(m, n, k, padding=1)]
if act: layers += [nn.ELU()]
return nn.Sequential(*layers)
def __init__(self,
in_dim,
num_labels,
layers,
out_dim,
input_kw=2,
input_dil=1,
positions=False,
softmax=False,
causal=False,
batch_norm=False):
"""
Constructor for WaveNetClassifier.
Args:
* num_features: python int; the number of channels in the featurized input sequence.
* feature_kwidth: python int; the kernel width of the featurization layer.
* num_labels: the number of labels in the softmax distribution at the output.
* layers: list of (non-causal) convolutional layers to stack. Each entry is of the form
(in_channels, out_channels, kernel_size, dilation).
* out_dim: the final dimension of the output before the dimensionality reduction to logits over labels.
* input_kw: size of the internal kernel of the conv1d going from input to conv-stack.
* input_dil: dilation for conv block going from input to conv-stack.
* positions: if True, use conv1x1 to mix position information to features.
* softmax: if True, softmax the output layer before returning. If False, return un-normalized sequence.
* causal: if True, use causal convolutions; otherwise use standard convolutions.
* batch_norm: if True, apply a batch-norm before each residual block.
"""
### parent constructor
super(RawCTCNet, self).__init__()
### attributes
self.in_dim = in_dim
self.num_labels = num_labels
self.layers = layers
self.num_layers = len(layers)
self.out_dim = out_dim
# input 1x1Conv layer:
self.input_kw = input_kw
self.input_dil = input_dil
# position-mixing on/off:
self.positions = positions
# softmax on/off:
self.softmax = softmax
# causal vs. standard convolutions:
self.causal = causal
# batch-norm on/off:
self.batch_norm = batch_norm
### submodules
# convolutional featurization layer:
self.feature_layer = nn.Sequential(
nn.Conv1d(in_dim, in_dim, kernel_size=1, padding=0, dilation=1),
nn.ELU(),
nn.Conv1d(in_dim, in_dim, kernel_size=1, padding=0, dilation=1),
nn.ELU())
# position-mixing bilinear layer:
if self.positions:
self.positions_conv1x1 = nn.Sequential(
nn.Conv1d(1, in_dim, kernel_size=1, padding=0, dilation=1),
nn.Hardtanh())
# input layer:
self.input_block = ResidualBlock(in_dim,
layers[0][0],
input_kw,
input_dil,
causal=self.causal)
self.input_skip_bottleneck = nn.Conv1d(layers[0][0],
out_dim,
kernel_size=1,
padding=0,
dilation=1)
# stack of residual convolutions and their bottlenecks for skip connections:
convolutions = []
skip_conn_bottlenecks = []
for (c_in, c_out, k, d) in layers:
convolutions.append(
ResidualBlock(c_in, c_out, k, d, causal=self.causal))
skip_conn_bottlenecks.append(
nn.Conv1d(c_out, out_dim, kernel_size=1, padding=0,
dilation=1))
self.convolutions = nn.ModuleList(convolutions)
self.bottlenecks = nn.ModuleList(skip_conn_bottlenecks)
# optional batch norms:
if self.batch_norm:
batch_norms = []
for (c_in, _, _, _) in layers:
batch_norms.append(nn.BatchNorm1d(c_in, affine=True))
self.batch_norms = nn.ModuleList(batch_norms)
# (1x1 Conv + ReLU + 1x1 Conv) stack, going from output dimension to logits over labels:
self.output_block = nn.Sequential(
nn.LeakyReLU(0.01),
nn.Conv1d(out_dim, out_dim, kernel_size=1, dilation=1),
nn.LeakyReLU(0.01),
nn.Conv1d(out_dim, num_labels, kernel_size=1, dilation=1))
### sensible initializations for parameters:
eps = 0.0001
if self.positions:
for p in self.positions_conv1x1.parameters():
if len(p.size()) > 1:
nn_init.eye(p.view(p.size(0), p.size(1)))
p.data.add_(torch.randn(p.size()).mul_(eps))
if len(p.size()) == 1:
p.data.zero_().add_(torch.randn(p.size()).mul_(eps))
for p in self.feature_layer.parameters():
if len(p.size()) > 1: nn_init.kaiming_uniform(p)
if len(p.size()) == 1:
p.data.zero_().add_(torch.randn(p.size()).mul_(eps))
for p in self.input_block.parameters():
if len(p.size()) > 1: nn_init.kaiming_uniform(p)
if len(p.size()) == 1:
p.data.zero_().add_(torch.randn(p.size()).mul_(eps))
for p in self.convolutions.parameters():
if len(p.size()) > 1: nn_init.kaiming_uniform(p)
if len(p.size()) == 1:
p.data.zero_().add_(torch.randn(p.size()).mul_(eps))
for p in self.bottlenecks.parameters():
if len(p.size()) > 1:
nn_init.eye(p.view(p.size(0), p.size(1)))
p.data.add_(torch.randn(p.size()).mul_(eps))
if len(p.size()) == 1:
p.data.zero_().add_(torch.randn(p.size()).mul_(eps))
for p in self.output_block.parameters():
if len(p.size()) > 1: nn_init.kaiming_uniform(p)
if len(p.size()) == 1:
p.data.zero_().add_(torch.randn(p.size()).mul_(eps))
def __init__(self, input_dim, output_dim):
super(_Aggregation, self).__init__()
self.aggre = nn.Sequential(
nn.Linear(input_dim, output_dim, bias=True),
nn.ELU(),
)
class Conv2dExtractor(nn.Module):
"""
Provides inteface of convolutional extractor.
Note:
Do not use this class directly, use one of the sub classes.
Define the 'self.conv' class variable.
Inputs: inputs, input_lengths
- **inputs** (batch, time, dim): Tensor containing input vectors
- **input_lengths**: Tensor containing containing sequence lengths
Returns: outputs, output_lengths
- **outputs**: Tensor produced by the convolution
- **output_lengths**: Tensor containing sequence lengths produced by the convolution
"""
supported_activations = {
'hardtanh': nn.Hardtanh(0, 20, inplace=True),
'relu': nn.ReLU(inplace=True),
'elu': nn.ELU(inplace=True),
'leaky_relu': nn.LeakyReLU(inplace=True),
'gelu': nn.GELU(),
'swish': Swish(),
}
def __init__(self, input_dim: int, activation: str = 'hardtanh') -> None:
super(Conv2dExtractor, self).__init__()
self.input_dim = input_dim
self.activation = Conv2dExtractor.supported_activations[activation]
self.conv = None
def get_output_lengths(self, seq_lengths: Tensor):
assert self.conv is not None, "self.conv should be defined"
for module in self.conv:
if isinstance(module, nn.Conv2d):
numerator = seq_lengths + 2 * module.padding[
1] - module.dilation[1] * (module.kernel_size[1] - 1) - 1
seq_lengths = numerator.float() / float(module.stride[1])
seq_lengths = seq_lengths.int() + 1
elif isinstance(module, nn.MaxPool2d):
seq_lengths >>= 1
return seq_lengths.int()
def get_output_dim(self):
if isinstance(self, VGGExtractor):
output_dim = (self.input_dim - 1
) << 5 if self.input_dim % 2 else self.input_dim << 5
elif isinstance(self, DeepSpeech2Extractor):
output_dim = int(math.floor(self.input_dim + 2 * 20 - 41) / 2 + 1)
output_dim = int(math.floor(output_dim + 2 * 10 - 21) / 2 + 1)
output_dim <<= 5
elif isinstance(self, Conv2dSubsampling):
factor = ((self.input_dim - 1) // 2 - 1) // 2
output_dim = self.out_channels * factor
else:
raise ValueError(f"Unsupported Extractor : {self.extractor}")
return output_dim
def forward(self, inputs: Tensor,
input_lengths: Tensor) -> Tuple[Tensor, Tensor]:
"""
inputs: torch.FloatTensor (batch, time, dimension)
input_lengths: torch.IntTensor (batch)
"""
outputs, output_lengths = self.conv(
inputs.unsqueeze(1).transpose(2, 3), input_lengths)
batch_size, channels, dimension, seq_lengths = outputs.size()
outputs = outputs.permute(0, 3, 1, 2)
outputs = outputs.view(batch_size, seq_lengths, channels * dimension)
return outputs, output_lengths
def __init__(self, in_chans,
n_classes,
final_conv_length='auto',
input_time_length=None,
pool_mode='mean',
f1=8,
d=2,
f2=16, # usually set to F1*D (?)
kernel_length=64,
third_kernel_size=(8, 4),
drop_prob=0.25,
siamese=False,
i_feature_alignment_layer=None # 0-based index modules
):
super(EEGNet, self).__init__()
if i_feature_alignment_layer is None:
i_feature_alignment_layer = 2 # default alignment layer
if final_conv_length == 'auto':
assert input_time_length is not None
# Assigns all parameters in init to self.param_name
self.__dict__.update(locals())
del self.self
# Define kind of pooling used:
pool_class = dict(max=nn.MaxPool2d, mean=nn.AvgPool2d)[self.pool_mode]
# Convolution accros temporal axis
self.temporal_conv = nn.Sequential(
# Rearrange dimensions, dimshuffle,
# tranform to shape required by pytorch:
Expression(_transpose_to_b_1_c_0),
# Temporal conv layer:
nn.Conv2d(in_channels=1, out_channels=self.f1,
kernel_size=(1, self.kernel_length),
stride=1,
bias=False,
padding=(0, self.kernel_length // 2)),
nn.BatchNorm2d(self.f1, momentum=0.01, affine=True, eps=1e-3)
)
self.spatial_conv = nn.Sequential(
# Spatial conv layer:
Conv2dWithConstraint(self.f1, self.f1 * self.d, (self.in_chans, 1),
max_norm=1, stride=1, bias=False,
groups=self.f1, padding=(0, 0)),
nn.BatchNorm2d(self.f1 * self.d, momentum=0.01, affine=True,
eps=1e-3),
nn.ELU(),
pool_class(kernel_size=(1, 4), stride=(1, 4))
)
self.separable_conv = nn.Sequential(
nn.Dropout(p=self.drop_prob),
# Separable conv layer:
nn.Conv2d(self.f1 * self.d, self.f1 * self.d, (1, 16), stride=1,
bias=False, groups=self.f1 * self.d,
padding=(0, 16 // 2)),
nn.Conv2d(self.f1 * self.d, self.f2, (1, 1), stride=1, bias=False,
padding=(0, 0)),
nn.BatchNorm2d(self.f2, momentum=0.01, affine=True, eps=1e-3),
nn.ELU(),
pool_class(kernel_size=(1, 8), stride=(1, 8))
)
out = np_to_var(
np.ones((1, self.in_chans, self.input_time_length, 1),
dtype=np.float32))
out = self.forward_init(out)
# out = self.separable_conv(self.spatial_conv(self.temporal_conv(out)))
n_out_virtual_chans = out.cpu().data.numpy().shape[2]
if self.final_conv_length == 'auto':
n_out_time = out.cpu().data.numpy().shape[3]
self.final_conv_length = n_out_time
# Classifier part:
self.cls = nn.Sequential(
nn.Dropout(p=self.drop_prob),
nn.Conv2d(self.f2, self.n_classes,
(n_out_virtual_chans, self.final_conv_length),
bias=True),
nn.LogSoftmax(dim=1),
# Transpose back to the the logic of _braindecode,
# so time in third dimension (axis=2)
# Transform back to original shape and
# squeeze to (batch_size, n_classes) size
Expression(_transpose_1_0),
Expression(_squeeze_final_output)
)
# Initialize weights of the network
self.apply(glorot_weight_zero_bias)
# Set feature space alignment layer, used in siamese training/testing
assert 0 <= self.i_feature_alignment_layer < len(self._modules), \
"Given feature space alignment layer does not " \
"exist for current model"
self.feature_alignment_layer = \
list(self._modules.items())[self.i_feature_alignment_layer][0]
def __init__(self, config, corpus_target, embReader):
super().__init__(config)
####
# init parameters
self.max_num_sents = config.max_num_sents # document length, in terms of the number of sentences
self.max_len_sent = config.max_len_sent # sentence length, in terms of words
self.max_len_doc = config.max_len_doc # document length, in terms of words
self.batch_size = config.batch_size
self.size_avg_pool_sent = config.size_avg_pool_sent
self.corpus_target = config.corpus_target
self.vocab = corpus_target.vocab # word2id
self.rev_vocab = corpus_target.rev_vocab # id2word
self.pad_id = corpus_target.pad_id
self.num_special_vocab = corpus_target.num_special_vocab
self.dropout_rate = config.dropout
self.rnn_cell_size = config.rnn_cell_size
self.output_size = config.output_size # the number of final output class
self.pad_level = config.pad_level
self.use_gpu = config.use_gpu
self.gen_logs = config.gen_logs
if not hasattr(config, "freeze_step"):
config.freeze_step = 5000
########
#
# self.base_encoder = Encoder_Coh(config, embReader)
self.base_encoder = Encoder_Main(config, embReader)
#
self.sim_cosine = torch.nn.CosineSimilarity(dim=2)
self.conv_sent = nn.Conv1d(in_channels=1,
out_channels=1,
kernel_size=3,
stride=2,
padding=1,
dilation=1,
groups=1,
bias=True)
# bias=False)
self.max_adapt_pool1_sent = nn.AdaptiveMaxPool1d(
self.size_avg_pool_sent)
#
fc_in_size = self.base_encoder.encoder_out_size + self.size_avg_pool_sent
linear_1_out = fc_in_size // 2
linear_2_out = linear_1_out // 2
self.linear_1 = nn.Linear(fc_in_size, linear_1_out)
nn.init.xavier_normal_(self.linear_1.weight)
self.linear_2 = nn.Linear(linear_1_out, linear_2_out)
nn.init.xavier_uniform_(self.linear_2.weight)
nn.init.xavier_normal_(self.linear_2.weight)
self.linear_out = nn.Linear(linear_2_out, self.output_size)
if corpus_target.output_bias is not None: # bias
init_mean_val = np.expand_dims(corpus_target.output_bias, axis=1)
bias_val = (np.log(init_mean_val) - np.log(1 - init_mean_val))
self.linear_out.bias.data = torch.from_numpy(bias_val).type(
torch.FloatTensor)
# nn.init.xavier_uniform_(self.linear_out.weight)
nn.init.xavier_normal_(self.linear_out.weight)
#
self.selu = nn.SELU()
self.elu = nn.ELU()
self.leak_relu = nn.LeakyReLU()
self.relu = nn.ReLU()
self.tanh = nn.Tanh()
self.sigmoid = nn.Sigmoid()
self.dropout_layer = nn.Dropout(self.dropout_rate)
self.dropout_01 = nn.Dropout(0.1)
self.dropout_02 = nn.Dropout(0.2)
self.softmax = nn.Softmax(dim=1)
return
def __init__(self, input_nc, output_nc, kernel_size, stride, padding, activation_func=nn.ELU()):
super(Conv, self).__init__()
self.conv = nn.Conv2d(in_channels=input_nc,
out_channels=output_nc,
kernel_size=kernel_size,
stride=stride,
padding=0,
bias=True)
self.activation_fn = activation_func
self.pad_fn = nn.ReplicationPad2d(padding)
def __init__(self, setting: dict):
super(PointCNN, self).__init__()
self.setting = setting
self.fts_is_None = self.setting['fts_is_None']
self.xconv_params = self.setting['xconv_params']
if not self.fts_is_None:
self.features_hd_generator=nn.Sequential(nn.Linear(self.setting['data_dim']-3,\
self.setting['xconv_params'][0]['C']),nn.ELU(),View((-1,self.setting['xconv_params'][0]['C'])),\
nn.BatchNorm1d(num_features=self.setting["xconv_params"][0]['C']),\
View((-1,setting["sample_num"],self.setting['xconv_params'][0]['C'])))
self.net = nn.Sequential()
self.with_X_transformation = setting['with_X_transformation']
self.fc_params = setting['fc_params']
self.links = []
self.sampling = setting['sampling']
prev_P = setting["sample_num"]
self.xconv_output_dim = []
param_dict = {
"xconv_output_dim": self.xconv_output_dim,
"xconv_params": self.setting["xconv_params"]
}
if "xdconv_params" in setting:
param_dict["xdconv_params"] = self.setting["xdconv_params"]
if self.fts_is_None:
self.xconv_output_dim.append(0)
else:
self.xconv_output_dim.append(self.setting['xconv_params'][0]['C'])
for layer_idx, layer_param in enumerate(self.setting['xconv_params']):
K = layer_param[
'K'] # K：在inverse density sampling时，需要根据每个点到与其最近的K个点的平均距离估算该点的概率密度，从而实现inverse denstiy sampling；这个K也是XConv操作里的K
D = layer_param['D'] # D：Dilation
if layer_param['P'] > 0:
P = layer_param['P'] # P：表示下一层要选用的样本点的数量
elif layer_param["P"] == -1:
P = prev_P
else:
print("P should either be positive integer or eqaul to -1!")
#exit()()
prev_P = P
C = layer_param['C']
with_global = (setting["with_global"]
and layer_idx == len(self.xconv_params) - 1)
self.links.append(layer_param['links'])
if layer_idx == 0:
C_pts_fts = C // 2 if self.fts_is_None else C // 4 # //表示除后向下取整数
depth_multiplier = 4
else:
C_prev = self.xconv_params[layer_idx - 1]['C']
C_pts_fts = C_prev // 4
depth_multiplier = math.ceil(C / C_prev)
last_layer_flag = (layer_idx == (
len(self.setting['xconv_params']) - 1))
self.net.add_module(
"layer{}".format(layer_idx),
module=XConv(setting=param_dict,
idx=layer_idx,
K=K,
dilation=D,
P=P,
C=C,
C_pts_fts=C_pts_fts,
depth_multiplier=depth_multiplier,
with_X_transfomation=self.with_X_transformation,
with_global=with_global,
sorting_method=None).to(config.device))
if last_layer_flag and with_global:
self.xconv_output_dim.append(C + C // 4)
else:
self.xconv_output_dim.append(C)
if "xdconv_params" in setting:
for layer_idx, layer_param in enumerate(setting["xdconv_params"]):
K = layer_param['K']
D = layer_param['D']
pts_layer_idx = layer_param['pts_layer_idx']
qrs_layer_idx = layer_param['qrs_layer_idx']
P = setting["xconv_params"][qrs_layer_idx]['P']
C = setting["xconv_params"][qrs_layer_idx]['C']
C_prev = setting["xconv_params"][pts_layer_idx]['C']
C_pts_fts = C_prev // 4
depth_multiplier = 1
with_global = False
last_layer_flag = (layer_idx == (
len(self.setting['xdconv_params']) - 1))
self.net.add_module(
"xdconv_layer{}".format(layer_idx),
module=XConv(
setting=param_dict,
idx=len(self.setting['xconv_params']) + layer_idx,
K=K,
dilation=D,
P=P,
C=C,
C_pts_fts=C_pts_fts,
depth_multiplier=depth_multiplier,
with_X_transfomation=self.with_X_transformation,
with_global=with_global,
sorting_method=None).to(config.device))
layer_name = "xdconv_dense{}".format(layer_idx)
self.net.add_module(
name=layer_name,
module=nn.Sequential(
nn.Linear(self.xconv_output_dim[qrs_layer_idx + 1] + C,
C).to(config.device), View((-1, C, P)),
nn.BatchNorm1d(num_features=C).to(config.device),
View((-1, P, C))).to(config.device))
self.xconv_output_dim.append(C)
for layer_idx, layer_param in enumerate(self.fc_params):
C = layer_param['C']
if layer_idx > 0:
if "xdconv_params" in setting:
P = setting["xconv_params"][setting["xdconv_params"][-1]
["qrs_layer_idx"]]["P"]
else:
P = setting["xconv_params"][-1]["P"]
self.net.add_module(name="dense{}".format(layer_idx),
module=nn.Sequential(
nn.Linear(prev_C, C), View((-1, C)),
nn.BatchNorm1d(num_features=C),
View((-1, P, C))).to(config.device))
else:
if "xdconv_params" in setting:
prev_C = setting["xconv_params"][
setting["xdconv_params"][-1]["qrs_layer_idx"]]["C"]
P = setting["xconv_params"][setting["xdconv_params"][-1]
["qrs_layer_idx"]]["P"]
else:
prev_C = self.setting["xconv_params"][-1]['C']
P = setting["xconv_params"][-1]["P"]
if self.setting["with_global"]:
prev_C += (prev_C // 4)
self.net.add_module(name="dense{}".format(layer_idx),
module=nn.Sequential(
nn.Linear(prev_C, C), View((-1, C)),
nn.BatchNorm1d(num_features=C),
View((-1, P, C))).to(config.device))
self.net.add_module(name="fc_dropout_{}".format(layer_idx),
module=nn.Dropout(
layer_param['dropout_rate']).to(
config.device))
prev_C = layer_param['C']
self.net.add_module(name="logits",
module=nn.Linear(self.fc_params[-1]["C"],
self.setting["num_classes"],
bias=True).to(config.device))
def __init__(self, input_nc, output_nc, scale, kernel_size, padding, activation_func=nn.ELU()):
super().__init__()
self.up = nn.Upsample(scale_factor = scale)
self.conv1 = Conv(input_nc, output_nc, kernel_size, 1, padding, activation_func)
def __init__(self, xs=(3, 32, 32), nz=1, zchannels=16, nprocessing=1, kernel_size=3, resdepth=2,
reswidth=256, dropout_p=0., tag='', root_process=True):
super().__init__()
# default: disable compressing mode
# if activated, tensors will be flattened
self.compressing = False
# hyperparameters
self.xs = xs
self.nz = nz
self.zchannels = zchannels
self.nprocessing = nprocessing
# latent height/width is always 16,
# the number of channels depends on the dataset
self.zdim = (self.zchannels, 16, 16)
self.resdepth = resdepth
self.reswidth = reswidth
self.kernel_size = kernel_size
# apply these two factors (i.e. on the ELBO) in sequence and it results in "bits/dim"
# factor to convert "nats" to bits
self.bitsscale = np.log2(np.e)
# factor to divide by the data dimension
self.perdimsscale = 1. / np.prod(self.xs)
# calculate processing layers convolutions options
# kernel/filter is 5, so in order to ensure same-size outputs, we have to pad by 2
padding_proc = (5 - 1) / 2
assert padding_proc.is_integer()
padding_proc = int(padding_proc)
# calculate other convolutions options
padding = (self.kernel_size - 1) / 2
assert padding.is_integer()
padding = int(padding)
# create loggers
self.tag = tag
if root_process:
current_time = datetime.now().strftime('%b%d_%H-%M-%S')
log_dir = os.path.join(
'runs/imagenet/', current_time + '_' + socket.gethostname() + tag)
self.log_dir = log_dir
self.logger = SummaryWriter(log_dir=self.log_dir)
# set-up current "best elbo"
self.best_elbo = np.inf
# distribute ResNet blocks over latent layers
resdepth = [0] * (self.nz)
i = 0
for _ in range(self.resdepth):
i = 0 if i == (self.nz) else i
resdepth[i] += 1
i += 1
# reduce initial variance of distributions corresponding
# to latent layers if latent nz increases
scale = 1.0 / (self.nz ** 0.5)
# activations
self.softplus = nn.Softplus()
self.sigmoid = nn.Sigmoid()
self.act = nn.ELU()
self.actresnet = nn.ELU()
# Below we build up the main model architecture of the inference- and generative-models
# All the architecure components are built up from different custom are existing PyTorch modules
# <===== INFERENCE MODEL =====>
# the bottom (zi=1) inference model
self.infer_in = nn.Sequential(
# shape: [1,32,32] -> [4,16,16]
modules.Squeeze2d(factor=2),
# shape: [4,16,16] -> [32,16,16]
modules.WnConv2d(4 * xs[0],
self.reswidth,
5,
1,
padding_proc,
init_scale=1.0,
loggain=True),
self.act
)
self.infer_res0 = nn.Sequential(
# shape: [32,16,16] -> [32,16,16]
modules.ResNetBlock(self.reswidth,
self.reswidth,
5,
1,
padding_proc,
self.nprocessing,
dropout_p,
self.actresnet),
self.act
) if self.nprocessing > 0 else modules.Pass()
self.infer_res1 = nn.Sequential(
# shape: [32,16,16] -> [32,16,16]
modules.ResNetBlock(self.reswidth,
self.reswidth,
self.kernel_size,
1,
padding,
resdepth[0],
dropout_p,
self.actresnet),
self.act
) if resdepth[0] > 0 else modules.Pass()
# shape: [32,16,16] -> [1,16,16]
self.infer_mu = modules.WnConv2d(self.reswidth,
self.zchannels,
self.kernel_size,
1,
padding,
init_scale=scale if self.nz > 1 else 2 ** 0.5 * scale)
# shape: [32,16,16] -> [1,16,16]
self.infer_std = modules.WnConv2d(self.reswidth,
self.zchannels,
self.kernel_size,
1,
padding,
init_scale=scale if self.nz > 1 else 2 ** 0.5 * scale)
# <===== DEEP INFERENCE MODEL =====>
# the deeper (zi > 1) inference models
self.deepinfer_in = nn.ModuleList([
# shape: [1,16,16] -> [32,16,16]
nn.Sequential(
modules.WnConv2d(self.zchannels,
self.reswidth,
self.kernel_size,
1,
padding,
init_scale=1.0,
loggain=True),
self.act
)
for _ in range(self.nz - 1)])
self.deepinfer_res = nn.ModuleList([
# shape: [32,16,16] -> [32,16,16]
nn.Sequential(
modules.ResNetBlock(self.reswidth,
self.reswidth,
self.kernel_size,
1,
padding,
resdepth[i + 1],
dropout_p,
self.actresnet),
self.act
) if resdepth[i + 1] > 0 else modules.Pass()
for i in range(self.nz - 1)])
self.deepinfer_mu = nn.ModuleList([
# shape: [32,16,16] -> [1,16,16]
nn.Sequential(
modules.WnConv2d(self.reswidth,
self.zchannels,
self.kernel_size,
1,
padding,
init_scale=scale if i < self.nz - 2 else 2 ** 0.5 * scale)
)
for i in range(self.nz - 1)])
self.deepinfer_std = nn.ModuleList([
# shape: [32,16,16] -> [1,16,16]
nn.Sequential(
modules.WnConv2d(self.reswidth,
self.zchannels,
self.kernel_size,
1,
padding,
init_scale=scale if i < self.nz - 2 else 2 ** 0.5 * scale)
)
for i in range(self.nz - 1)])
# <===== DEEP GENERATIVE MODEL =====>
# the deeper (zi > 1) generative models
self.deepgen_in = nn.ModuleList([
# shape: [1,16,16] -> [32,16,16]
nn.Sequential(
modules.WnConv2d(self.zchannels,
self.reswidth,
self.kernel_size,
1,
padding,
init_scale=1.0,
loggain=True),
self.act
)
for _ in range(self.nz - 1)])
self.deepgen_res = nn.ModuleList([
# shape: [32,16,16] -> [32,16,16]
nn.Sequential(
modules.ResNetBlock(self.reswidth,
self.reswidth,
self.kernel_size,
1,
padding,
resdepth[i + 1],
dropout_p,
self.actresnet),
self.act
) if resdepth[i + 1] > 0 else modules.Pass()
for i in range(self.nz - 1)])
self.deepgen_mu = nn.ModuleList([
# shape: [32,16,16] -> [1,16,16]
nn.Sequential(
modules.WnConv2d(self.reswidth,
self.zchannels,
self.kernel_size,
1,
padding,
init_scale=scale)
)
for _ in range(self.nz - 1)])
self.deepgen_std = nn.ModuleList([
# shape: [32,16,16] -> [1,16,16]
nn.Sequential(
modules.WnConv2d(self.reswidth,
self.zchannels,
self.kernel_size,
1,
padding, init_scale=scale)
)
for _ in range(self.nz - 1)])
# <===== GENERATIVE MODEL =====>
# the bottom (zi = 1) inference model
self.gen_in = nn.Sequential(
# shape: [1,16,16] -> [32,16,16]
modules.WnConv2d(self.zchannels,
self.reswidth,
self.kernel_size,
1,
padding,
init_scale=1.0,
loggain=True),
self.act
)
self.gen_res1 = nn.Sequential(
# shape: [32,16,16] -> [32,16,16]
modules.ResNetBlock(self.reswidth,
self.reswidth,
self.kernel_size,
1,
padding,
resdepth[0],
dropout_p,
self.actresnet),
self.act
) if resdepth[0] > 0 else modules.Pass()
self.gen_res0 = nn.Sequential(
# shape: [32,16,16] -> [32,16,16]
modules.ResNetBlock(self.reswidth,
self.reswidth,
5,
1,
padding_proc,
self.nprocessing,
dropout_p,
self.actresnet),
self.act
) if self.nprocessing > 0 else modules.Pass()
self.gen_mu = nn.Sequential(
# shape: [32,16,16] -> [4,16,16]
modules.WnConv2d(self.reswidth,
4 * xs[0],
self.kernel_size,
1,
padding,
init_scale=0.1),
# shape: [4,16,16] -> [1,32,23]
modules.UnSqueeze2d(factor=2)
)
# the scale parameter of the bottom (zi = 1) generative model is modelled unconditional
self.gen_std = nn.Parameter(torch.Tensor(*self.xs))
nn.init.zeros_(self.gen_std)
def __init__(self, config):
super().__init__()
self.config = config
self.batch_norm = nn.BatchNorm2d(1)
self.conv_stem = nn.Sequential(
nn.Conv2d(1, 30, (3, 3), bias=True),
nn.ELU(),
GaussianDropout(0.1),
GaussianNoise(0.1),
nn.Conv2d(30, 30, (1, 35), bias=True),
nn.ELU(),
GaussianDropout(0.1),
GaussianNoise(0.1),
nn.Conv2d(30, 30, (7, 1), bias=True),
nn.ELU(),
GaussianDropout(0.1),
GaussianNoise(0.1)
)
self.note_frames = nn.Sequential(
nn.Conv2d(30, 10, (3, 3), bias=True),
nn.ELU(),
GaussianDropout(0.5),
GaussianNoise(0.1),
Flatten(),
nn.Linear(1060, 88)
)
self.note_onsets = nn.Sequential(
nn.Conv2d(30, 10, (3, 3), bias=True),
nn.ELU(),
GaussianDropout(0.5),
GaussianNoise(0.1),
Flatten(),
nn.Linear(1060, 88)
)
self.note_offsets = nn.Sequential(
nn.Conv2d(30, 10, (3, 3), bias=True),
nn.ELU(),
GaussianDropout(0.5),
GaussianNoise(0.1),
Flatten(),
nn.Linear(1060, 88)
)
for m in self.modules():
if isinstance(m, nn.Conv2d):
init.xavier_uniform_(m.weight)
if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
init.xavier_uniform_(m.weight)
if m.bias is not None:
m.bias.data.zero_()
elif isinstance(m, nn.BatchNorm2d) or isinstance(m, nn.BatchNorm1d):
m.weight.data.fill_(1)
m.bias.data.zero_()
self.best = dict(
f=0,
loss=10.
)
def ELUCons(elu, nchan):
if elu:
return nn.ELU(inplace=True)
else:
return nn.PReLU(nchan)
def __init__(self, in_channels, out_channels):
super(ConvBlock, self).__init__()
self.conv = Conv3x3(in_channels, out_channels)
self.nonlin = nn.ELU(inplace=True)
def __init__(
self,
shape,
n_class,
channel,
kernel_size,
n_block,
n_res_block,
res_channel,
attention=True,
dropout=0.1,
n_cond_res_block=0,
cond_res_channel=0,
cond_res_kernel=3,
n_out_res_block=0,
):
super().__init__()
height, width = shape
self.n_class = n_class
if kernel_size % 2 == 0:
kernel = kernel_size + 1
else:
kernel = kernel_size
self.horizontal = CausalConv2d(n_class,
channel, [kernel // 2, kernel],
padding='down')
self.vertical = CausalConv2d(n_class,
channel, [(kernel + 1) // 2, kernel // 2],
padding='downright')
coord_x = (torch.arange(height).float() - height / 2) / height
coord_x = coord_x.view(1, 1, height, 1).expand(1, 1, height, width)
coord_y = (torch.arange(width).float() - width / 2) / width
coord_y = coord_y.view(1, 1, 1, width).expand(1, 1, height, width)
self.register_buffer('background', torch.cat([coord_x, coord_y], 1))
self.blocks = nn.ModuleList()
for i in range(n_block):
self.blocks.append(
PixelBlock(
channel,
res_channel,
kernel_size,
n_res_block,
attention=attention,
dropout=dropout,
condition_dim=cond_res_channel,
))
if n_cond_res_block > 0:
self.cond_resnet = CondResNet(n_class, cond_res_channel,
cond_res_kernel, n_cond_res_block)
out = []
for i in range(n_out_res_block):
out.append(GatedResBlock(channel, res_channel, 1))
out.extend([nn.ELU(inplace=True), WNConv2d(channel, n_class, 1)])
self.out = nn.Sequential(*out)
def __init__(
self,
obs_spaces: collections.OrderedDict,
recurrent: bool = False,
hidden_size: int = 64,
use_critic: bool = True,
critic_detach: bool = True,
):
num_inputs = 0
self.obs_keys = obs_spaces.keys()
self.image_space = obs_spaces['image']
mlp_obs_spaces = obs_spaces.copy()
mlp_obs_spaces.update({
'image':
spaces.Box(
low=0.0, # Arbitrary value
high=1.0, # Arbitrary value
shape=(hidden_size, ),
dtype='float',
)
})
self.mlp_obs_keys = mlp_obs_spaces.keys()
super().__init__(
obs_spaces=mlp_obs_spaces,
recurrent=recurrent,
hidden_size=hidden_size,
use_critic=use_critic,
critic_detach=critic_detach,
)
_neg_slope = 0.1
init_ = lambda m: init(
m, nn.init.orthogonal_, lambda x: nn.init.constant_(x, 0),
nn.init.calculate_gain('leaky_relu', param=_neg_slope))
NEW_CNN = True
H, W, num_channels = self.image_space.shape
if NEW_CNN:
self.cnn = nn.Sequential(
init_(nn.Conv2d(num_channels, 16, (3, 3), padding=1)),
nn.ReLU(),
# nn.MaxPool2d((2, 2)),
init_(nn.Conv2d(16, 32, (2, 2))),
nn.ReLU(),
init_(nn.Conv2d(32, 64, (2, 2))),
nn.ReLU(),
Flatten(),
)
else:
self.cnn = nn.Sequential(
init_(nn.Conv2d(num_channels, 16, 1, stride=1)),
# nn.LeakyReLU(_neg_slope),
nn.ELU(),
init_(nn.Conv2d(16, 8, 3, stride=1, padding=2)),
# nn.LeakyReLU(_neg_slope),
nn.ELU(),
# init_(nn.Conv2d(64, 64, 5, stride=1, padding=2)),
# nn.LeakyReLU(_neg_slope),
Flatten(),
)
output_h_w, out_channels = utils.conv_sequential_output_shape((H, W),
self.cnn)
h_w_prod = output_h_w[0] * output_h_w[1]
self.fc = nn.Sequential(
init_(nn.Linear(out_channels * h_w_prod, hidden_size)),
# nn.LeakyReLU(_neg_slope),
# nn.ELU(),
)
self.apply(initialize_parameters)
def __init__(self,
model,
image_size=(340, 480),
finetune=False,
restore=RESTORE_FROM,
n_layers=2,
n_heads=NUM_CLASSES,
softmax=True):
super(SegAttention, self).__init__()
self.image_h, self.image_w = image_size
self.seg_net = Seg_Model(num_classes=NUM_CLASSES)
self.seg_net.eval()
self.softmax = nn.Softmax(dim=1) if softmax else None
self.semantic_drop = nn.Dropout2d(0.3)
try:
self.cnn = model(pretrained=True).features
except AttributeError:
self.cnn = nn.Sequential(*list(model(
pretrained=True).children())[:-3])
if not finetune:
for param in self.cnn.parameters(): # freeze cnn params
param.requires_grad = False
self.n_layers = n_layers
self.n_heads = n_heads
self.hidden_dim = NUM_CLASSES
if restore is not None:
self.seg_net.load_state_dict(
torch.load(restore, map_location=device))
for param in self.seg_net.parameters(): # freeze segnet params
param.requires_grad = False
sample = torch.randn([3, self.image_h, self.image_w]).unsqueeze(0)
self.seg_dims = self.seg_net(
sample)[0].size() # for layer size definitionlayers
self.cnn_size = self.cnn(sample).size()
# self.upsample = nn.Upsample((self.seg_dims[2], self.seg_dims[3]))
self.upsample = nn.ConvTranspose2d(self.cnn_size[1],
self.cnn_size[1],
kernel_size=3,
stride=2,
padding=1,
dilation=1)
self.cnn_size = self.upsample(self.cnn(sample)).size()
self.attentions = nn.ModuleList([
nn.MultiheadAttention(embed_dim=self.hidden_dim,
num_heads=self.n_heads,
dropout=0.1,
kdim=NUM_CLASSES,
vdim=self.cnn_size[1],
qdim=self.cnn_size[1])
for _ in range(self.n_layers)
])
self.fc = nn.Linear(
self.hidden_dim * self.seg_dims[2] * self.seg_dims[3], 256)
self.activation = nn.ELU()
self.dropout = nn.Dropout(0.1)
self.output = nn.Linear(256, 1)
def __init__(self,
input_dim,
output_dim,
resample=None,
act=nn.ELU(),
normalization=nn.BatchNorm2d,
adjust_padding=False,
dilation=None,
spec_norm=False):
super().__init__()
self.non_linearity = act
self.input_dim = input_dim
self.output_dim = output_dim
self.resample = resample
self.normalization = normalization
if resample == 'down':
if dilation is not None:
self.conv1 = dilated_conv3x3(input_dim,
input_dim,
dilation=dilation,
spec_norm=spec_norm)
self.normalize2 = normalization(input_dim)
self.conv2 = dilated_conv3x3(input_dim,
output_dim,
dilation=dilation,
spec_norm=spec_norm)
conv_shortcut = partial(dilated_conv3x3,
dilation=dilation,
spec_norm=spec_norm)
else:
self.conv1 = conv3x3(input_dim, input_dim, spec_norm=spec_norm)
self.normalize2 = normalization(input_dim)
self.conv2 = ConvMeanPool(input_dim,
output_dim,
3,
adjust_padding=adjust_padding,
spec_norm=spec_norm)
conv_shortcut = partial(ConvMeanPool,
kernel_size=1,
adjust_padding=adjust_padding,
spec_norm=spec_norm)
elif resample is None:
if dilation is not None:
conv_shortcut = partial(dilated_conv3x3,
dilation=dilation,
spec_norm=spec_norm)
self.conv1 = dilated_conv3x3(input_dim,
output_dim,
dilation=dilation,
spec_norm=spec_norm)
self.normalize2 = normalization(output_dim)
self.conv2 = dilated_conv3x3(output_dim,
output_dim,
dilation=dilation,
spec_norm=spec_norm)
else:
# conv_shortcut = nn.Conv2d ### Something wierd here.
conv_shortcut = partial(conv1x1, spec_norm=spec_norm)
self.conv1 = conv3x3(input_dim,
output_dim,
spec_norm=spec_norm)
self.normalize2 = normalization(output_dim)
self.conv2 = conv3x3(output_dim,
output_dim,
spec_norm=spec_norm)
else:
raise Exception('invalid resample value')
if output_dim != input_dim or resample is not None:
self.shortcut = conv_shortcut(input_dim, output_dim)
self.normalize1 = normalization(input_dim)
def test_elu(self):
x = Variable(torch.randn(1, 2, 3, 4), requires_grad=True)
self.assertONNX(nn.ELU(), x)
def __init__(self,
nb_words=[],
nb_authors=0,
freq_feature=[],
nb_fcat=0,
freq_masking=[]):
super(BinaryMeasurer, self).__init__()
#nb_bi, nb_tri, nb_tetra = nb_words
#print(nb_words)
self.freq_feature = freq_feature
self.nb_fcat = nb_fcat
self.freq_masking = freq_masking
#nb_words_limits = [s.squeeze().size(0) for x in self.freq_masking for s in x]
#print(len(freq_feature[0]))
#quit()
#self.n_neurons = 1502#*4*2
#self.n_neurons_2 = 600
#self.fc1 = nn.Linear(self.n_neurons,self.n_neurons_2//2)
#self.fc2 = nn.Linear(self.n_neurons_2//2,self.n_neurons_2//2)
#if use_cuda:
# module
#self.list_modules = []
#for cat in range(nb_fcat):
nw = sum([x // 10 for x in nb_words])
#print(nw)
#quit()
self.layer1 = nn.ModuleList(
[nn.Linear(nb_words[r], nb_words[r]) for r in range(nb_fcat)])
self.layer2 = nn.ModuleList(
[nn.Linear(nb_words[r], nb_words[r]) for r in range(nb_fcat)])
self.layer3 = nn.ModuleList(
[nn.Linear(nb_words[r], nb_words[r]) for r in range(nb_fcat)])
#self.fl = nn.ModuleList([nn.Linear(nb_words[r],nb_authors) for r in range(nb_fcat)])
self.par = nn.Parameter(torch.FloatTensor(
[1.0])) # for r in range(nb_fcat)])
#self.attn2 = nn.Parameter(torch.randn(nw)/nw)# for r in range(nb_fcat)])
#self.fc1 = nn.Linear(len(nb_words),nb_authors)
self.fc2 = nn.Linear(nb_authors * 2, nb_authors)
self.fc3 = nn.Linear(nw, nb_authors)
self.fc4 = nn.Linear(nw, nb_authors)
#self.conv1 = nn.Conv1d(1,nb_authors,nw, padding=0)
#kernel_size_2 = (nw+2*2-(5-1)-1)+1
#self.conv2 = nn.Conv1d(nb_authors,nb_authors,1, padding=0)
#(nw+2*2-(5-1)-1)+1
#self.fc5 = nn.Linear(nb_bi,nb_bi)
#self.fc6 = nn.Linear(nb_bi,nb_bi)
#self.fc7 = nn.Linear(nb_bi,nb_bi)
#self.fc8 = nn.Linear(nb_bi,nb_bi)
#self.fc9 = nn.Linear(nb_tri,nb_tri)
#self.fc10 = nn.Linear(nb_tri,nb_tri)
#self.fc11 = nn.Linear(nb_tri,nb_tri)
#self.fc12 = nn.Linear(nb_tri,nb_tri)
#self.fc13 = nn.Linear(nb_tetra,nb_tetra)
#self.fc14 = nn.Linear(nb_tetra,nb_tetra)
#self.fc15 = nn.Linear(nb_tetra,nb_tetra)
#self.fc16 = nn.Linear(nb_tetra,nb_tetra)
self.dropout1 = nn.Dropout(0.5)
self.dropout1_2 = nn.Dropout(0.0)
self.dropout2 = nn.Dropout(0.5)
self.relu = nn.ReLU()
self.elu = nn.ELU()
self.tanh = nn.Tanh()
self.sigmoid = nn.Sigmoid()
self.softmax_ = nn.Softmax(dim=0)
self.softmax = nn.LogSoftmax(dim=0)
def __init__(self, in_ch, out_ch, dim):
super(ConvBlock, self).__init__()
self.conv = getattr(nn, 'Conv{}d'.format(dim))(in_ch, out_ch, K_D, stride=dim, padding=K_D // 2)
self.bnrm = getattr(nn, 'BatchNorm{}d'.format(dim))(out_ch)
self.drop = nn.Sequential(perm(), nn.Dropout2d(DR_S), perm()) if dim == 1 else nn.Dropout2d(DR_S)
self.block = nn.Sequential(self.conv, nn.ELU(), self.bnrm, self.drop)
def deconv_block(in_dim, out_dim):
return nn.Sequential(
nn.Conv2d(in_dim, out_dim, kernel_size=3, stride=1, padding=1),
nn.ELU(True),
nn.Conv2d(out_dim, out_dim, kernel_size=3, stride=1, padding=1),
nn.ELU(True), nn.UpsamplingNearest2d(scale_factor=2))
def __init__(self, num_classes, encoder=None):
super(Net, self).__init__()
self.num_classes = num_classes
self.conv1 = nn.Conv2d(3, 96, kernel_size=3, stride=2, padding=1) # 32
# self.bn1 = nn.BatchNorm2d(96)
self.relu1 = nn.ELU(inplace=True)
self.maxpool1 = nn.MaxPool2d(kernel_size=2, stride=2) # 16
self.fire1_1 = Fire(96, 16, 64)
self.fire1_2 = Fire(128, 16, 64)
self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2) # 8
self.fire2_1 = Fire(128, 32, 128)
self.fire2_2 = Fire(256, 32, 128)
self.maxpool3 = nn.MaxPool2d(kernel_size=2, stride=2) # 4
self.fire3_1 = Fire(256, 64, 256)
self.fire3_2 = Fire(512, 64, 256)
self.fire3_3 = Fire(512, 64, 256)
self.parallel = ParallelDilatedConv(512, 512)
self.deconv1 = nn.ConvTranspose2d(512,
256,
3,
stride=2,
padding=1,
output_padding=1)
# self.bn2 = nn.BatchNorm2d(256)
self.relu2 = nn.ELU(inplace=True)
self.deconv2 = nn.ConvTranspose2d(512,
128,
3,
stride=2,
padding=1,
output_padding=1)
# self.bn3 = nn.BatchNorm2d(128)
self.relu3 = nn.ELU(inplace=True)
self.deconv3 = nn.ConvTranspose2d(256,
96,
3,
stride=2,
padding=1,
output_padding=1)
# self.bn4 = nn.BatchNorm2d(96)
self.relu4 = nn.ELU(inplace=True)
self.deconv4 = nn.ConvTranspose2d(192,
self.num_classes,
3,
stride=2,
padding=1,
output_padding=1)
self.conv3_1 = nn.Conv2d(256, 256, kernel_size=3, stride=1,
padding=1) # 32
self.conv3_2 = nn.Conv2d(512, 512, kernel_size=3, stride=1,
padding=1) # 32
self.conv2_1 = nn.Conv2d(128, 128, kernel_size=3, stride=1,
padding=1) # 32
self.conv2_2 = nn.Conv2d(256, 256, kernel_size=3, stride=1,
padding=1) # 32
self.conv1_1 = nn.Conv2d(96, 96, kernel_size=3, stride=1,
padding=1) # 32
self.conv1_2 = nn.Conv2d(192, 192, kernel_size=3, stride=1,
padding=1) # 32
self.relu1_1 = nn.ELU(inplace=True)
self.relu1_2 = nn.ELU(inplace=True)
self.relu2_1 = nn.ELU(inplace=True)
self.relu2_2 = nn.ELU(inplace=True)
self.relu3_1 = nn.ELU(inplace=True)
self.relu3_2 = nn.ELU(inplace=True)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.in_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()