def decode(self, trg_input, src_map, oov_list, enc_context, enc_hidden, trg_mask, ctx_mask):
'''
:param
trg_input: (batch_size, trg_len)
src_map : (batch_size, src_len), almost the same with src but oov words are replaced with temporary oov index, for copy mechanism to map the probs of pointed words to vocab words. The word index can be beyond vocab_size, e.g. 50000, 50001, 50002 etc, depends on how many oov words appear in the source text
context vector: (batch_size, src_len, hidden_size * num_direction) the outputs (hidden vectors) of encoder
:returns
decoder_probs : (batch_size, trg_seq_len, vocab_size + max_oov_number)
decoder_outputs : (batch_size, trg_seq_len, hidden_size)
attn_weights : (batch_size, trg_seq_len, src_seq_len)
copy_attn_weights : (batch_size, trg_seq_len, src_seq_len)
'''
batch_size = trg_input.size(0)
src_len = enc_context.size(1)
trg_len = trg_input.size(1)
context_dim = enc_context.size(2)
trg_hidden_dim = self.trg_hidden_dim
# prepare the init hidden vector, (batch_size, dec_hidden_dim) -> 2 * (1, batch_size, dec_hidden_dim)
init_hidden = self.init_decoder_state(enc_hidden[0], enc_hidden[1])
# enc_context has to be reshaped before dot attention (batch_size, src_len, context_dim) -> (batch_size, src_len, trg_hidden_dim)
enc_context = nn.Tanh()(self.encoder2decoder_hidden(enc_context.contiguous().view(-1, context_dim))).view(batch_size, src_len, trg_hidden_dim)
# maximum length to unroll
max_length = trg_input.size(1) - 1
# Teacher Forcing
self.current_batch += 1
if self.do_teacher_forcing():
logging.info("Training batches with Teacher Forcing")
'''
Normal RNN procedure
'''
# truncate the last word, as there's no further word after it for decoder to predict
trg_input = trg_input[:, :-1]
# initialize target embedding and reshape the targets to be time step first
trg_emb = self.embedding(trg_input) # (batch_size, trg_len, embed_dim)
trg_emb = trg_emb.permute(1, 0, 2) # (trg_len, batch_size, embed_dim)
# both in/output of decoder LSTM is batch-second (trg_len, batch_size, trg_hidden_dim)
decoder_outputs, hidden = self.decoder(
trg_emb, init_hidden
)
# Get the h_tilde (batch_size, trg_len, trg_hidden_dim) and attention weights (batch_size, trg_len, src_len)
h_tildes, attn_weights, attn_logits = self.attention_layer(decoder_outputs.permute(1, 0, 2), enc_context)
# compute the output decode_logit and read-out as probs: p_x = Softmax(W_s * h_tilde), (batch_size, trg_len, trg_hidden_size) -> (batch_size * trg_len, vocab_size)
# h_tildes=(batch_size, trg_len, trg_hidden_size) -> decoder2vocab(h_tildes.view)=(batch_size * trg_len, vocab_size) -> decoder_logits=(batch_size, trg_len, vocab_size)
decoder_logits = self.decoder2vocab(h_tildes.view(-1, trg_hidden_dim)).view(batch_size, max_length, -1)
'''
Copy Mechanism
'''
# copy_weights and copy_logits is (batch_size, trg_len, src_len)
if self.copy_attention_layer:
_, copy_weights, copy_logits = self.copy_attention_layer(decoder_outputs.permute(1, 0, 2), enc_context)
else:
copy_logits = attn_logits
# merge the generative and copying probs, (batch_size, trg_len, vocab_size + max_oov_number)
decoder_log_probs = self.merge_copy_probs(decoder_logits, copy_logits, src_map, oov_list) # (batch_size, trg_len, vocab_size + max_oov_number)
decoder_outputs = decoder_outputs.permute(1, 0, 2) # (batch_size, trg_len, trg_hidden_dim)
else:
logging.info("Training batches with All Sampling")
'''
Normal RNN procedure
'''
# take the first word (should be BOS <s>) of each target sequence (batch_size, 1)
trg_input = trg_input[:, 0].unsqueeze(1)
decoder_log_probs = []
decoder_outputs= []
attn_weights = []
copy_weights = []
for di in range(max_length):
# initialize target embedding and reshape the targets to be time step first
trg_emb = self.embedding(trg_input) # (batch_size, 1, embed_dim)
trg_emb = trg_emb.permute(1, 0, 2) # (1, batch_size, embed_dim)
# this is trg_len first
decoder_output, hidden = self.decoder(
trg_emb, init_hidden
)
# Get the h_tilde (hidden after attention) and attention weights. h_tilde (batch_size,1,trg_hidden), attn_weight & attn_logit(batch_size,1,src_len)
h_tilde, attn_weight, attn_logit = self.attention_layer(decoder_output.permute(1, 0, 2), enc_context)
# compute the output decode_logit and read-out as probs: p_x = Softmax(W_s * h_tilde)
# h_tilde=(batch_size, 1, trg_hidden_size) -> decoder2vocab(h_tilde.view)=(batch_size * 1, vocab_size) -> decoder_logit=(batch_size, 1, vocab_size)
decoder_logit = self.decoder2vocab(h_tilde.view(-1, trg_hidden_dim)).view(batch_size, 1, -1)
'''
Copy Mechanism
'''
# copy_weights and copy_logits is (batch_size, trg_len, src_len)
if self.copy_attention_layer:
_, copy_weight, copy_logit = self.copy_attention_layer(decoder_output.permute(1, 0, 2), enc_context)
else:
copy_weight = attn_weight
copy_logit = attn_logit
# merge the generative and copying probs (batch_size, 1, vocab_size + max_oov_number)
decoder_log_prob = self.merge_copy_probs(decoder_logit, copy_logit, src_map, oov_list)
'''
Find the next word
'''
# (deprecated, should not merge)before locating the topk, we need to move the probs of oovs to <unk>
# oov2unk_prob = self.merge_oov2unk(decoder_log_prob, max_oov_number)
top_v, top_idx = decoder_log_prob.data.topk(1, dim=-1)
# replace the oov words to <unk>
top_idx[top_idx >= self.vocab_size] = self.unk_word
top_idx = Variable(top_idx.squeeze(2))
# top_idx and next_index are (batch_size, 1)
trg_input = top_idx.cuda() if torch.cuda.is_available() else top_idx
# permute to trg_len first, otherwise the cat operation would mess up things
decoder_log_probs.append(decoder_log_prob.permute(1, 0, 2))
decoder_outputs.append(decoder_output)
attn_weights.append(attn_weight.permute(1, 0, 2))
copy_weights.append(copy_weight.permute(1, 0, 2))
# convert output into the right shape and make batch first
decoder_log_probs = torch.cat(decoder_log_probs, 0).permute(1, 0, 2) # (batch_size, trg_seq_len, vocab_size + max_oov_number)
decoder_outputs = torch.cat(decoder_outputs, 0).permute(1, 0, 2) # (batch_size, trg_seq_len, hidden_size)
attn_weights = torch.cat(attn_weights, 0).permute(1, 0, 2) # (batch_size, trg_seq_len, src_seq_len)
copy_weights = torch.cat(copy_weights, 0).permute(1, 0, 2) # (batch_size, trg_seq_len, src_seq_len)
# Return final outputs (logits after log_softmax), hidden states, and attention weights (for visualization)
return decoder_log_probs, decoder_outputs, attn_weights, copy_weights
def __init__(self, input_nc, output_nc, ngf=32, n_downsampling=4, norm_layer=nn.BatchNorm2d):
super(Encoder, self).__init__()
self.output_nc = output_nc
model = [
nn.ReflectionPad2d(3),
nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0),
norm_layer(ngf),
nn.ReLU(True),
]
### downsample
for i in range(n_downsampling):
mult = 2 ** i
model += [
nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3, stride=2, padding=1),
norm_layer(ngf * mult * 2),
nn.ReLU(True),
]
### upsample
for i in range(n_downsampling):
mult = 2 ** (n_downsampling - i)
model += [
nn.ConvTranspose2d(
ngf * mult, int(ngf * mult / 2), kernel_size=3, stride=2, padding=1, output_padding=1
),
norm_layer(int(ngf * mult / 2)),
nn.ReLU(True),
]
model += [nn.ReflectionPad2d(3), nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0), nn.Tanh()]
self.model = nn.Sequential(*model)
def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, gpu_ids=[], padding_type='reflect'):
assert(n_blocks >= 0)
super(ResnetGeneratorMMReverse, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.gpu_ids = gpu_ids
if type(norm_layer) == functools.partial:
use_bias = norm_layer.func == nn.InstanceNorm2d
else:
use_bias = norm_layer == nn.InstanceNorm2d
model = [nn.ReflectionPad2d(3),
nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0,
bias=use_bias),
norm_layer(ngf),
nn.ReLU(True)]
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
model += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3,
stride=2, padding=1, bias=use_bias),
norm_layer(ngf * mult * 2),
nn.ReLU(True)]
pre_f_blocks = 4
pre_l_blocks = 7
mult = 2**n_downsampling
for i in range(pre_f_blocks):
model += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
model_pre = []
for i in range(pre_f_blocks,pre_l_blocks):
model_pre += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
model_post1 = []
model_post2 = []
for i in range(pre_l_blocks,n_blocks):
model_post1 += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
model_post2 += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
model_post1 += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),
kernel_size=3, stride=2,
padding=1, output_padding=1,
bias=use_bias),
norm_layer(int(ngf * mult / 2)),
nn.ReLU(True)]
model_post2 += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),
kernel_size=3, stride=2,
padding=1, output_padding=1,
bias=use_bias),
norm_layer(int(ngf * mult / 2)),
nn.ReLU(True)]
model_post1 += [nn.ReflectionPad2d(3)]
model_post1 += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]
model_post1 += [nn.Tanh()]
model_post2 += [nn.ReflectionPad2d(3)]
model_post2 += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]
model_post2 += [nn.Tanh()]
self.model_post1 = nn.Sequential(*model_post1)
self.model_post2 = nn.Sequential(*model_post2)
self.model_pre = nn.Sequential(*model_pre)
self.model = nn.Sequential(*model)
def __init__(self,
vocabs,
opt,
predictor_tgt=None,
predictor_src=None,
PreModelClass='TransformerPredictor'):
super().__init__(vocabs=vocabs, opt=opt)
if not predictor_tgt:
if opt.load_pred_target:
predictor_tgt = eval(PreModelClass).from_file(
opt.load_pred_target, opt)
else:
predictor_tgt = eval(PreModelClass)(vocabs,
opt,
predict_inverse=False)
if not predictor_src:
if opt.load_pred_source:
predictor_src = eval(PreModelClass).from_file(
opt.load_pred_source, opt)
else:
predictor_src = eval(PreModelClass)(vocabs,
opt,
predict_inverse=True)
if opt.token_level:
if predictor_src:
predictor_src.vocabs = vocabs
if predictor_tgt:
predictor_tgt.vocabs = vocabs
self.predictor_tgt = predictor_tgt
self.predictor_src = predictor_src
self.mlp = None
self.sentence_pred = None
self.sentence_sigma = None
self.lstm_input_size = 2 * opt.hidden_pred + opt.out_embeddings_size
if opt.mlp_est:
self.mlp = nn.Sequential(
nn.Linear(self.lstm_input_size, opt.hidden_est), nn.Tanh())
self.lstm_input_size = opt.hidden_est
self.lstm = nn.LSTM(
input_size=self.lstm_input_size,
hidden_size=self.opt.hidden_est,
num_layers=self.opt.rnn_layers_est,
batch_first=True,
dropout=self.opt.dropout_est,
bidirectional=True,
)
sentence_input_size = 2 * opt.rnn_layers_est * opt.hidden_est
self.sentence_pred = nn.Sequential(
nn.Linear(sentence_input_size, sentence_input_size // 2),
nn.Sigmoid(),
nn.Linear(sentence_input_size // 2, sentence_input_size // 4),
nn.Sigmoid(),
nn.Linear(sentence_input_size // 4, 1),
)
if self.opt.sentence_ll:
# Predict truncated Gaussian distribution
self.sentence_sigma = nn.Sequential(
nn.Linear(sentence_input_size, sentence_input_size // 2),
nn.Sigmoid(),
nn.Linear(sentence_input_size // 2, sentence_input_size // 4),
nn.Sigmoid(),
nn.Linear(sentence_input_size // 4, 1),
nn.Sigmoid(),
)
self.mse_loss = nn.MSELoss(reduction='sum')
if opt.start_stop:
self.start_PreQEFV = nn.Parameter(
torch.zeros(1, 1, opt.out_embeddings_size))
self.end_PreQEFV = nn.Parameter(
torch.zeros(1, 1, opt.out_embeddings_size))
self.opt = opt
def __init__(self,
cell='gru',
use_baseline=True,
n_actions=10,
n_units=64,
fusion_dim=128,
n_input=76,
n_hidden=128,
demo_dim=17,
n_output=1,
dropout=0.0,
lamda=0.5,
device='cpu'):
super(Agent, self).__init__()
self.cell = cell
self.use_baseline = use_baseline
self.n_actions = n_actions
self.n_units = n_units
self.n_input = n_input
self.n_hidden = n_hidden
self.n_output = n_output
self.dropout = dropout
self.lamda = lamda
self.fusion_dim = fusion_dim
self.demo_dim = demo_dim
self.device = device
self.agent1_action = []
self.agent1_prob = []
self.agent1_entropy = []
self.agent1_baseline = []
self.agent2_action = []
self.agent2_prob = []
self.agent2_entropy = []
self.agent2_baseline = []
self.agent1_fc1 = nn.Linear(self.n_hidden + self.demo_dim,
self.n_units)
self.agent2_fc1 = nn.Linear(self.n_input + self.demo_dim, self.n_units)
self.agent1_fc2 = nn.Linear(self.n_units, self.n_actions)
self.agent2_fc2 = nn.Linear(self.n_units, self.n_actions)
if use_baseline == True:
self.agent1_value = nn.Linear(self.n_units, 1)
self.agent2_value = nn.Linear(self.n_units, 1)
if self.cell == 'lstm':
self.rnn = nn.LSTMCell(self.n_input, self.n_hidden)
else:
self.rnn = nn.GRUCell(self.n_input, self.n_hidden)
for name, param in self.rnn.named_parameters():
if 'bias' in name:
nn.init.constant(param, 0.0)
elif 'weight' in name:
nn.init.orthogonal_(param)
if dropout > 0.0:
self.nn_dropout = nn.Dropout(p=dropout)
self.init_h = nn.Linear(self.demo_dim, self.n_hidden)
self.init_c = nn.Linear(self.demo_dim, self.n_hidden)
self.fusion = nn.Linear(self.n_hidden + self.demo_dim, self.fusion_dim)
self.output = nn.Linear(self.fusion_dim, self.n_output)
self.sigmoid = nn.Sigmoid()
self.softmax = nn.Softmax()
self.tanh = nn.Tanh()
self.relu = nn.ReLU()
out = (out > 0.5) * 1
return out.data.numpy()
# 绘制出决策面
plot_decision_boundary(lambda x: plot_network(x), x.numpy(), y.numpy())
plt.title("two layer network")
plt.show()
"""下面介绍Sequential与Module"""
# Sequential 允许我们构建序列化的模块，而 Module 是一种更加灵活的模型定义方式
# Sequential方法构建模型
seq_net = nn.Sequential(
nn.Linear(2, 4), # 输入层是2个神经元，隐藏层是4个神经元 xw + b
nn.Tanh(),
nn.Linear(4, 1) # 隐藏层是4个神经元，输出层是1个神经元
)
# 序列模块可以通过索引访问每一层
print(seq_net[0])
# 打印出第一层的权重
print(seq_net[0].weight)
# 通过parameters可以获得模型的参数
param = seq_net.parameters()
# 定义优化算法
optimizer = optim.SGD(param, 1.)
# 训练10000次
for epoch in range(10000):
out = seq_net(Variable(x))
loss = loss_func(out, Variable(y))
optimizer.zero_grad()
def __init__(self, cfg):
super(VqaModelDncQC, self).__init__()
self.cfg = cfg
self.img_encoder = ImgEncoder(
cfg["hyperparameters"]["commun_embed_size"])
self.qst_encoder = QstEncoderDnc(cfg)
if cfg["dnc_c"]["nonlinearity"] == "tanh":
self.nonlinearity = nn.Tanh()
elif cfg["dnc_c"]["nonlinearity"] == "relu":
self.nonlinearity = nn.ReLU()
elif cfg["dnc_c"]["nonlinearity"] == "sigmoid":
self.nonlinearity = nn.Sigmoid()
else:
raise ValueError(
"<{}> is not a valid non-linearity function.".format(
cfg["dnc_c"]["nonlinearity"]))
self.tanh = nn.Tanh()
self.dropout = nn.Dropout(cfg["hyperparameters"]["dropout"])
if cfg["dnc_c"]["type"] == "MLP":
self.dnc = DNC_MLP(
input_size=cfg["hyperparameters"]["commun_embed_size"],
output_size=cfg["dnc_c"]["output_size"],
hidden_size=cfg["dnc_c"]["hidden_dim"],
num_hidden_layers=cfg["dnc_c"]["num_layers_hidden"],
dropout=cfg["dnc_c"]["dropout"],
nr_cells=cfg["dnc_c"]["n"],
cell_size=cfg["dnc_c"]["w"],
read_heads=cfg["dnc_c"]["r"],
nonlinearity=self.nonlinearity,
gpu_id=cfg["hyperparameters"]["gpu_id"],
debug=cfg["dnc_c"]["debug"],
clip=20,
)
elif cfg["dnc_c"]["type"] == "LSTM":
self.dnc = DNC(
input_size=cfg["hyperparameters"]["commun_embed_size"],
output_size=cfg["dnc_c"]["output_size"],
hidden_size=cfg["dnc_c"]["hidden_dim"],
rnn_type=cfg["dnc_c"]["rnn_type"],
num_layers=cfg["dnc_c"]["num_layers"],
num_hidden_layers=cfg["dnc_c"]["num_layers_hidden"],
bias=True,
batch_first=True,
dropout=cfg["dnc_c"]["dropout"],
bidirectional=cfg["dnc_c"]["bidirectional"],
nr_cells=cfg["dnc_c"]["n"],
cell_size=cfg["dnc_c"]["w"],
read_heads=cfg["dnc_c"]["r"],
gpu_id=cfg["hyperparameters"]["gpu_id"],
independent_linears=True,
share_memory=True,
debug=cfg["dnc_c"]["debug"],
clip=20)
else:
raise ValueError("dnc controller type <{}> is not defined".format(
cfg["dnc"]["dnc_c_type"]))
if cfg["dnc_c"]["concat_out_rv"]:
in_fc_1 = cfg["dnc_c"][
"output_size"] + cfg["dnc_c"]["w"] * cfg["dnc_c"]["r"]
else:
in_fc_1 = cfg["dnc_c"]["output_size"]
self.fc_1 = nn.Linear(in_fc_1,
cfg["hyperparameters"]["ans_vocab_size"])
self.fc_2 = nn.Linear(cfg["hyperparameters"]["ans_vocab_size"],
cfg["hyperparameters"]["ans_vocab_size"])
def __init__(self, input_size, hidden_size, output_size):
super(TwoLayerLRSeq, self).__init__()
self.model = nn.Sequential(nn.Linear(input_size, hidden_size),
nn.Tanh(),
nn.Linear(hidden_size, output_size))
def __init__(self, input_size, use_stn=False, use_attention=False):
super(PPG2ECG, self).__init__()
self.use_stn = use_stn
self.use_attention = use_attention
# build main transformer
self.main = nn.Sequential(
# encoder
nn.Conv1d(1, 32, kernel_size=31, stride=2, padding=15),
nn.PReLU(32),
nn.Conv1d(32, 64, 31, 1, 15),
nn.PReLU(64),
nn.Conv1d(64, 128, 31, 2, 15),
nn.PReLU(128),
nn.Conv1d(128, 256, 31, 1, 15),
nn.PReLU(256),
nn.Conv1d(256, 512, 31, 2, 15),
nn.PReLU(512),
# decoder
nn.ConvTranspose1d(
512, 256, kernel_size=31, stride=2,
padding=15, output_padding=1),
nn.PReLU(256),
nn.ConvTranspose1d(256, 128, 31, 1, 15),
nn.PReLU(128),
nn.ConvTranspose1d(128, 64, 31, 2, 15, 1),
nn.PReLU(64),
nn.ConvTranspose1d(64, 32, 31, 1, 15),
nn.PReLU(32),
nn.ConvTranspose1d(32, 1, 31, 2, 15, 1),
nn.Tanh(),
)
# build stn (optional)
if use_stn:
# pylint: disable=not-callable
self.restriction = torch.tensor(
[1, 0, 0, 0], dtype=torch.float, requires_grad=False)
self.register_buffer('restriction_const', self.restriction)
self.stn_conv = nn.Sequential(
nn.Conv1d(
in_channels=1, out_channels=8, kernel_size=7, stride=1),
nn.MaxPool1d(kernel_size=2, stride=2),
nn.Conv1d(
in_channels=8, out_channels=10, kernel_size=5, stride=1),
nn.MaxPool1d(kernel_size=2, stride=2),
)
n_stn_conv = self.get_stn_conv_out(input_size)
self.stn_fc = nn.Sequential(
Flatten(),
nn.Linear(n_stn_conv, 32),
nn.ReLU(True),
nn.Linear(32, 4)
)
self.stn_fc[3].weight.data.zero_()
self.stn_fc[3].bias.data = torch.FloatTensor([1, 0, 1, 0])
# build attention network (optional)
if use_attention:
self.attn = nn.Sequential(
nn.Linear(input_size, input_size),
nn.ReLU(),
nn.Linear(input_size, input_size)
)
self.attn_len = input_size
def __init__(self,
input_nc,
output_nc,
ngf=64,
norm_layer=nn.BatchNorm3d,
use_dropout=False,
n_blocks=6,
padding_type='replicate'):
"""Construct a Resnet-based generator
Parameters:
input_nc (int) -- the number of channels in input images
output_nc (int) -- the number of channels in output images
ngf (int) -- the number of filters in the last conv layer
norm_layer -- normalization layer
use_dropout (bool) -- if use dropout layers
n_blocks (int) -- the number of ResNet blocks
padding_type (str) -- the name of padding layer in conv layers: reflect | replicate | zero
"""
assert (n_blocks >= 0)
super(ResnetGenerator, self).__init__()
if type(norm_layer) == functools.partial:
use_bias = norm_layer.func == nn.InstanceNorm3d
else:
use_bias = norm_layer == nn.InstanceNorm3d
model = [
nn.ReplicationPad3d(3),
nn.Conv3d(input_nc, ngf, kernel_size=7, padding=0, bias=use_bias),
norm_layer(ngf),
nn.ReLU(True)
]
n_downsampling = 2
for i in range(n_downsampling): # add downsampling layers
mult = 3**i
model += [
nn.Conv3d(ngf * mult,
ngf * mult * 3,
kernel_size=3,
stride=2,
padding=1,
bias=use_bias),
norm_layer(ngf * mult * 3),
nn.ReLU(True)
]
mult = 3**n_downsampling
for i in range(n_blocks): # add ResNet blocks
model += [
ResnetBlock3D(ngf * mult,
padding_type=padding_type,
norm_layer=norm_layer,
use_dropout=use_dropout,
use_bias=use_bias)
]
for i in range(n_downsampling): # add upsampling layers
mult = 3**(n_downsampling - i)
model += [
nn.ConvTranspose3d(ngf * mult,
int(ngf * mult / 2),
kernel_size=3,
stride=2,
padding=1,
output_padding=1,
bias=use_bias),
norm_layer(int(ngf * mult / 2)),
nn.ReLU(True)
]
model += [nn.ReplicationPad3d(3)]
model += [nn.Conv3d(ngf, output_nc, kernel_size=7, padding=0)]
model += [nn.Tanh()]
self.model = nn.Sequential(*model)
def __init__(self, input_size, hidden_size, output_size):
super(TwoLayerLR, self).__init__()
self.fc1 = nn.Linear(input_size, hidden_size)
self.tanh = nn.Tanh()
self.fc2 = nn.Linear(hidden_size, output_size)
def __init__(self,
outer_nc,
inner_nc,
input_nc=None,
submodule=None,
outermost=False,
innermost=False,
norm_layer=nn.BatchNorm2d,
use_dropout=False):
"""Construct a Unet submodule with skip connections.
Parameters:
outer_nc (int) -- the number of filters in the outer conv layer
inner_nc (int) -- the number of filters in the inner conv layer
input_nc (int) -- the number of channels in input images/features
submodule (UnetSkipConnectionBlock) -- previously defined submodules
outermost (bool) -- if this module is the outermost module
innermost (bool) -- if this module is the innermost module
norm_layer -- normalization layer
use_dropout (bool) -- if use dropout layers.
"""
super(UnetSkipConnectionBlock, self).__init__()
self.outermost = outermost
if type(norm_layer) == functools.partial:
use_bias = norm_layer.func == nn.InstanceNorm2d
else:
use_bias = norm_layer == nn.InstanceNorm2d
if input_nc is None:
input_nc = outer_nc
downconv = nn.Conv2d(input_nc,
inner_nc,
kernel_size=4,
stride=2,
padding=1,
bias=use_bias)
downrelu = nn.LeakyReLU(0.2, True)
downnorm = norm_layer(inner_nc)
uprelu = nn.ReLU(True)
upnorm = norm_layer(outer_nc)
if outermost:
upconv = nn.ConvTranspose2d(inner_nc * 2,
outer_nc,
kernel_size=4,
stride=2,
padding=1)
down = [downconv]
up = [uprelu, upconv, nn.Tanh()]
model = down + [submodule] + up
elif innermost:
upconv = nn.ConvTranspose2d(inner_nc,
outer_nc,
kernel_size=4,
stride=2,
padding=1,
bias=use_bias)
down = [downrelu, downconv]
up = [uprelu, upconv, upnorm]
model = down + up
else:
upconv = nn.ConvTranspose2d(inner_nc * 2,
outer_nc,
kernel_size=4,
stride=2,
padding=1,
bias=use_bias)
down = [downrelu, downconv, downnorm]
up = [uprelu, upconv, upnorm]
if use_dropout:
model = down + [submodule] + up + [nn.Dropout(0.5)]
else:
model = down + [submodule] + up
self.model = nn.Sequential(*model)
def forward(self, input):
tanh = nn.Tanh()
height_batch = self.preprocess(input)
height_batch = height_batch.view(-1, 4 * self.ngf, 4, 4)
_4x4 = height_batch
_8x8 = self._4_to_8(_4x4)
_16x16 = self._8_to_16(_8x8)
upsample = nn.Upsample(size=(32, 32), mode='bilinear')
height_batch = (tanh(self._16_to_32(_16x16)) + \
upsample(tanh(self._16_to_16(_16x16))) + \
upsample(tanh(self._8_to_8(_8x8))) + \
upsample(tanh(self._4_to_4(_4x4)))) / 4.0
height_batch = height_batch.permute(0, 3, 2, 1)
if np.any(np.isnan(height_batch.data.numpy())):
print('NANNANNAN')
exit()
if self.save_heightfield:
height_batch_np = height_batch.data.numpy()
height_flatten = np.zeros([32 * 8, 32 * 8, 1])
for i in range(8):
for j in range(8):
img = height_batch_np[8 * i + j, :, :, :]
height_flatten[32 * i:32 * (i + 1),
32 * j:32 * (j + 1), :] = img
image.imwrite(
height_flatten.squeeze(),
'results/heightfield_gan/heightfield_%06d.png' % iteration)
output = Variable(torch.zeros([input.shape[0], 1, 32, 32]))
for i in range(input.shape[0]):
height = torch.stack([\
Variable(torch.from_numpy(np.zeros(heightfield_res, dtype=np.float32))),
height_batch[i, :, :, 0],
Variable(torch.from_numpy(np.zeros(heightfield_res, dtype=np.float32)))],
dim=-1)
height = height.view([-1, 3])
shape_plane.vertices = plane_vertices + height
if self.save_heightfield:
v = shape_plane.vertices.data.numpy()
ind = shape_plane.indices.data.numpy() + 1
with open('results/heightfield_gan/model_%06d_%03d.obj' \
% (self.iteration, i), 'w') as f:
for vid in range(v.shape[0]):
f.write('v %f %f %f\n' %
(v[vid, 0], v[vid, 1], v[vid, 2]))
for iid in range(ind.shape[0]):
f.write('f %d %d %d\n' %
(ind[iid, 0], ind[iid, 1], ind[iid, 2]))
shape_plane.normals = compute_vertex_normal(
shape_plane.vertices, shape_plane.indices)
cam = camera.Camera(\
position = Variable(torch.from_numpy(np.array([self.xz[i][0], 3, self.xz[i][1]], dtype=np.float32))),
look_at = Variable(torch.from_numpy(np.array([0, 0, 0], dtype=np.float32))),
up = Variable(torch.from_numpy(np.array([0, 1, 0], dtype=np.float32))),
cam_to_world = None,
fov = Variable(torch.from_numpy(np.array([45.0], dtype=np.float32))),
clip_near = Variable(torch.from_numpy(np.array([0.01], dtype=np.float32))),
clip_far = Variable(torch.from_numpy(np.array([10000.0], dtype=np.float32))),
resolution = self.resolution)
args = render_pytorch.RenderFunction.serialize_scene(\
cam,materials,shapes,lights,self.resolution,4,1)
render = render_pytorch.RenderFunction.apply
img = render(random.randint(0, 1048576), *args)
img = img.permute([2, 1, 0])
output[i, :, :, :] = img[0, :, :]
return output
def __init__(self, input_size):
super().__init__()
self.dense = nn.Linear(input_size, input_size)
self.activation = nn.Tanh()
def __init__(
self,
in_channels: int,
latent_dim: int,
hidden_dims: List = None,
hidden_dims2: List = None,
lr: float = 0.005,
weight_decay: Optional[float] = 0,
scheduler_gamma: Optional[float] = 0.95,
) -> None:
super(TwoStageVAE, self).__init__(lr=lr,
weight_decay=weight_decay,
scheduler_gamma=scheduler_gamma)
self.latent_dim = latent_dim
modules = []
if hidden_dims is None:
hidden_dims = [32, 64, 128, 256, 512]
if hidden_dims2 is None:
hidden_dims2 = [1024, 1024]
# Build Encoder
for h_dim in hidden_dims:
modules.append(
nn.Sequential(
nn.Conv2d(
in_channels,
out_channels=h_dim,
kernel_size=3,
stride=2,
padding=1,
),
nn.BatchNorm2d(h_dim),
nn.LeakyReLU(),
))
in_channels = h_dim
self.encoder = nn.Sequential(*modules)
self.fc_mu = nn.Linear(hidden_dims[-1] * 4, latent_dim)
self.fc_var = nn.Linear(hidden_dims[-1] * 4, latent_dim)
# Build Decoder
modules = []
self.decoder_input = nn.Linear(latent_dim, hidden_dims[-1] * 4)
hidden_dims.reverse()
for i in range(len(hidden_dims) - 1):
modules.append(
nn.Sequential(
nn.ConvTranspose2d(
hidden_dims[i],
hidden_dims[i + 1],
kernel_size=3,
stride=2,
padding=1,
output_padding=1,
),
nn.BatchNorm2d(hidden_dims[i + 1]),
nn.LeakyReLU(),
))
self.decoder = nn.Sequential(*modules)
self.final_layer = nn.Sequential(
nn.ConvTranspose2d(
hidden_dims[-1],
hidden_dims[-1],
kernel_size=3,
stride=2,
padding=1,
output_padding=1,
),
nn.BatchNorm2d(hidden_dims[-1]),
nn.LeakyReLU(),
nn.Conv2d(hidden_dims[-1],
out_channels=3,
kernel_size=3,
padding=1),
nn.Tanh(),
)
# ---------------------- Second VAE ---------------------------#
encoder2 = []
in_channels = self.latent_dim
for h_dim in hidden_dims2:
encoder2.append(
nn.Sequential(nn.Linear(in_channels, h_dim),
nn.BatchNorm1d(h_dim), nn.LeakyReLU()))
in_channels = h_dim
self.encoder2 = nn.Sequential(*encoder2)
self.fc_mu2 = nn.Linear(hidden_dims2[-1], self.latent_dim)
self.fc_var2 = nn.Linear(hidden_dims2[-1], self.latent_dim)
decoder2 = []
hidden_dims2.reverse()
in_channels = self.latent_dim
for h_dim in hidden_dims2:
decoder2.append(
nn.Sequential(nn.Linear(in_channels, h_dim),
nn.BatchNorm1d(h_dim), nn.LeakyReLU()))
in_channels = h_dim
self.decoder2 = nn.Sequential(*decoder2)
def __init__(self, norm_layer=nn.BatchNorm2d, classes=529):
super(SIGGRAPHGenerator, self).__init__()
# Conv1
model1 = [
nn.Conv2d(4, 64, kernel_size=3, stride=1, padding=1, bias=True),
]
model1 += [
nn.ReLU(True),
]
model1 += [
nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=True),
]
model1 += [
nn.ReLU(True),
]
model1 += [
norm_layer(64),
]
# add a subsampling operation
# Conv2
model2 = [
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1, bias=True),
]
model2 += [
nn.ReLU(True),
]
model2 += [
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1, bias=True),
]
model2 += [
nn.ReLU(True),
]
model2 += [
norm_layer(128),
]
# add a subsampling layer operation
# Conv3
model3 = [
nn.Conv2d(128, 256, kernel_size=3, stride=1, padding=1, bias=True),
]
model3 += [
nn.ReLU(True),
]
model3 += [
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=True),
]
model3 += [
nn.ReLU(True),
]
model3 += [
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=True),
]
model3 += [
nn.ReLU(True),
]
model3 += [
norm_layer(256),
]
# add a subsampling layer operation
# Conv4
model4 = [
nn.Conv2d(256, 512, kernel_size=3, stride=1, padding=1, bias=True),
]
model4 += [
nn.ReLU(True),
]
model4 += [
nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1, bias=True),
]
model4 += [
nn.ReLU(True),
]
model4 += [
nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1, bias=True),
]
model4 += [
nn.ReLU(True),
]
model4 += [
norm_layer(512),
]
# Conv5
model5 = [
nn.Conv2d(512,
512,
kernel_size=3,
dilation=2,
stride=1,
padding=2,
bias=True),
]
model5 += [
nn.ReLU(True),
]
model5 += [
nn.Conv2d(512,
512,
kernel_size=3,
dilation=2,
stride=1,
padding=2,
bias=True),
]
model5 += [
nn.ReLU(True),
]
model5 += [
nn.Conv2d(512,
512,
kernel_size=3,
dilation=2,
stride=1,
padding=2,
bias=True),
]
model5 += [
nn.ReLU(True),
]
model5 += [
norm_layer(512),
]
# Conv6
model6 = [
nn.Conv2d(512,
512,
kernel_size=3,
dilation=2,
stride=1,
padding=2,
bias=True),
]
model6 += [
nn.ReLU(True),
]
model6 += [
nn.Conv2d(512,
512,
kernel_size=3,
dilation=2,
stride=1,
padding=2,
bias=True),
]
model6 += [
nn.ReLU(True),
]
model6 += [
nn.Conv2d(512,
512,
kernel_size=3,
dilation=2,
stride=1,
padding=2,
bias=True),
]
model6 += [
nn.ReLU(True),
]
model6 += [
norm_layer(512),
]
# Conv7
model7 = [
nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1, bias=True),
]
model7 += [
nn.ReLU(True),
]
model7 += [
nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1, bias=True),
]
model7 += [
nn.ReLU(True),
]
model7 += [
nn.Conv2d(512, 512, kernel_size=3, stride=1, padding=1, bias=True),
]
model7 += [
nn.ReLU(True),
]
model7 += [
norm_layer(512),
]
# Conv7
model8up = [
nn.ConvTranspose2d(512,
256,
kernel_size=4,
stride=2,
padding=1,
bias=True)
]
model3short8 = [
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=True),
]
model8 = [
nn.ReLU(True),
]
model8 += [
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=True),
]
model8 += [
nn.ReLU(True),
]
model8 += [
nn.Conv2d(256, 256, kernel_size=3, stride=1, padding=1, bias=True),
]
model8 += [
nn.ReLU(True),
]
model8 += [
norm_layer(256),
]
# Conv9
model9up = [
nn.ConvTranspose2d(256,
128,
kernel_size=4,
stride=2,
padding=1,
bias=True),
]
model2short9 = [
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1, bias=True),
]
# add the two feature maps above
model9 = [
nn.ReLU(True),
]
model9 += [
nn.Conv2d(128, 128, kernel_size=3, stride=1, padding=1, bias=True),
]
model9 += [
nn.ReLU(True),
]
model9 += [
norm_layer(128),
]
# Conv10
model10up = [
nn.ConvTranspose2d(128,
128,
kernel_size=4,
stride=2,
padding=1,
bias=True),
]
model1short10 = [
nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1, bias=True),
]
# add the two feature maps above
model10 = [
nn.ReLU(True),
]
model10 += [
nn.Conv2d(128,
128,
kernel_size=3,
dilation=1,
stride=1,
padding=1,
bias=True),
]
model10 += [
nn.LeakyReLU(negative_slope=.2),
]
# classification output
model_class = [
nn.Conv2d(256,
classes,
kernel_size=1,
padding=0,
dilation=1,
stride=1,
bias=True),
]
# regression output
model_out = [
nn.Conv2d(128,
2,
kernel_size=1,
padding=0,
dilation=1,
stride=1,
bias=True),
]
model_out += [nn.Tanh()]
self.model1 = nn.Sequential(*model1)
self.model2 = nn.Sequential(*model2)
self.model3 = nn.Sequential(*model3)
self.model4 = nn.Sequential(*model4)
self.model5 = nn.Sequential(*model5)
self.model6 = nn.Sequential(*model6)
self.model7 = nn.Sequential(*model7)
self.model8up = nn.Sequential(*model8up)
self.model8 = nn.Sequential(*model8)
self.model9up = nn.Sequential(*model9up)
self.model9 = nn.Sequential(*model9)
self.model10up = nn.Sequential(*model10up)
self.model10 = nn.Sequential(*model10)
self.model3short8 = nn.Sequential(*model3short8)
self.model2short9 = nn.Sequential(*model2short9)
self.model1short10 = nn.Sequential(*model1short10)
self.model_class = nn.Sequential(*model_class)
self.model_out = nn.Sequential(*model_out)
self.upsample4 = nn.Sequential(*[
nn.Upsample(scale_factor=4, mode='bilinear'),
])
self.softmax = nn.Sequential(*[
nn.Softmax(dim=1),
])
def __init__(self, config):
super().__init__()
self.dense = nn.Linear(config.hidden_size, config.hidden_size)
self.activation = nn.Tanh()
import numpy as np
import csv
import math
import os.path
import timeit
from collections import deque
import pickle
from multiprocessing import Pool
N, D_in, H, D_out = 4361, 22, 100, 10
# = Variable(torch.zeros(N, D_in),requires_grad=False)
#y = Variable(torch.zeros(N, 3), requires_grad=False)
relu = nn.ReLU()
sig = nn.Sigmoid()
tanh = nn.Tanh()
loss_fn = nn.MSELoss()
learning_rate = 0.0001
class CarData:
def __init__(self, dataList):
self.outputdata = []
self.sensordata = []
self.outputdata.append(dataList[0])
self.outputdata.append(dataList[1])
self.outputdata.append(dataList[2])
for i in range(3, len(dataList)):
self.sensordata.append(dataList[i])
def get_output_data(self):
def __init__(self, num_input, num_hidden, num_output):
super(Module_model, self).__init__()
self.layer1 = nn.Linear(num_input, num_hidden)
self.layer2 = nn.Tanh()
self.layer3 = nn.Linear(num_hidden, num_output)
def __init__(self, out_dim):
super(SAP, self).__init__()
# Setup
self.act_fn = nn.Tanh()
self.sap_layer = SelfAttentionPooling(out_dim)
def __init__(self,
img_size=224,
patch_size=16,
in_chans=3,
num_classes=1000,
embed_dim=768,
depth=12,
num_heads=12,
mlp_ratio=4.,
qkv_bias=True,
qk_scale=None,
representation_size=None,
drop_rate=0.,
attn_drop_rate=0.,
drop_path_rate=0.,
hybrid_backbone=None,
norm_layer=None):
"""
Args:
img_size (int, tuple): input image size
patch_size (int, tuple): patch size
in_chans (int): number of input channels
num_classes (int): number of classes for classification head
embed_dim (int): embedding dimension
depth (int): depth of transformer
num_heads (int): number of attention heads
mlp_ratio (int): ratio of mlp hidden dim to embedding dim
qkv_bias (bool): enable bias for qkv if True
qk_scale (float): override default qk scale of head_dim ** -0.5 if set
representation_size (Optional[int]): enable and set representation layer (pre-logits) to this value if set
drop_rate (float): dropout rate
attn_drop_rate (float): attention dropout rate
drop_path_rate (float): stochastic depth rate
hybrid_backbone (nn.Module): CNN backbone to use in-place of PatchEmbed module
norm_layer: (nn.Module): normalization layer
"""
super().__init__()
self.num_classes = num_classes
self.num_features = self.embed_dim = embed_dim # num_features for consistency with other models
norm_layer = norm_layer or partial(nn.LayerNorm, eps=1e-6)
if hybrid_backbone is not None:
self.patch_embed = HybridEmbed(hybrid_backbone,
img_size=img_size,
in_chans=in_chans,
embed_dim=embed_dim)
else:
self.patch_embed = PatchEmbed(img_size=img_size,
patch_size=patch_size,
in_chans=in_chans,
embed_dim=embed_dim)
num_patches = self.patch_embed.num_patches
self.cls_token = nn.Parameter(torch.zeros(1, 1, embed_dim))
self.pos_embed = nn.Parameter(
torch.zeros(1, num_patches + 1, embed_dim))
self.pos_drop = nn.Dropout(p=drop_rate)
dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)
] # stochastic depth decay rule
self.blocks = nn.ModuleList([
Block(dim=embed_dim,
num_heads=num_heads,
mlp_ratio=mlp_ratio,
qkv_bias=qkv_bias,
qk_scale=qk_scale,
drop=drop_rate,
attn_drop=attn_drop_rate,
drop_path=dpr[i],
norm_layer=norm_layer) for i in range(depth)
])
self.norm = norm_layer(embed_dim)
# Representation layer
if representation_size:
self.num_features = representation_size
self.pre_logits = nn.Sequential(
OrderedDict([('fc', nn.Linear(embed_dim, representation_size)),
('act', nn.Tanh())]))
else:
self.pre_logits = nn.Identity()
# Classifier head
self.head = nn.Linear(
self.num_features,
num_classes) if num_classes > 0 else nn.Identity()
trunc_normal_(self.pos_embed, std=.02)
trunc_normal_(self.cls_token, std=.02)
self.apply(self._init_weights)
def __init__(self, conv_dim=64):
super(Generator, self).__init__()
input_dim = 256
curr_dim = input_dim
# Makeup representation fully connected layer
layers_makeup = []
layers_makeup.append(nn.Linear(32, 256))
layers_makeup.append(nn.ReLU(inplace=True))
layers_makeup.append(nn.Linear(256, 256))
layers_makeup.append(nn.ReLU(inplace=True))
layers_makeup.append(nn.Linear(256, 512))
self.makeup = nn.Sequential(*layers_makeup)
# Pose and Face blend
layers_blend = []
for i in range(4):
layers_blend.append(
ResidualBlock(dim_in=input_dim * 2, dim_out=input_dim * 2))
layers_blend.append(
nn.Conv2d(input_dim * 2,
input_dim,
kernel_size=3,
stride=1,
padding=1,
bias=False))
layers_blend.append(nn.InstanceNorm2d(input_dim, affine=True))
layers_blend.append(nn.ReLU(inplace=True))
self.blend = nn.Sequential(*layers_blend)
# Main Generator
self.res_1 = ResidualBlock(dim_in=input_dim, dim_out=input_dim)
self.res_2 = ResidualBlock(dim_in=input_dim, dim_out=input_dim)
self.res_3 = ResidualBlock(dim_in=input_dim, dim_out=input_dim)
self.res_4 = ResidualBlock(dim_in=input_dim, dim_out=input_dim)
# Up-Sampling
layers = []
for i in range(2):
layers.append(
nn.ConvTranspose2d(curr_dim,
curr_dim // 2,
kernel_size=4,
stride=2,
padding=1,
bias=False))
layers.append(nn.InstanceNorm2d(curr_dim // 2, affine=True))
layers.append(nn.ReLU(inplace=True))
curr_dim = curr_dim // 2
layers.append(
nn.Conv2d(curr_dim,
3,
kernel_size=7,
stride=1,
padding=3,
bias=False))
layers.append(nn.InstanceNorm2d(3, affine=True))
layers.append(nn.Tanh())
self.main = nn.Sequential(*layers)
def __init__(self,
outer_nc,
inner_nc,
input_nc=None,
submodule=None,
outermost=False,
innermost=False,
norm_layer=nn.BatchNorm2d,
use_dropout=False):
super(UnetSkipConnectionBlock, self).__init__()
self.outermost = outermost
if type(norm_layer) == functools.partial:
use_bias = norm_layer.func == nn.InstanceNorm2d
else:
use_bias = norm_layer == nn.InstanceNorm2d
if input_nc is None:
input_nc = outer_nc
downconv = nn.Conv2d(input_nc,
inner_nc,
kernel_size=4,
stride=2,
padding=1,
bias=use_bias)
downrelu = nn.LeakyReLU(0.2, True)
downnorm = norm_layer(inner_nc)
uprelu = nn.ReLU(True)
upnorm = norm_layer(outer_nc)
if outermost:
upconv = nn.ConvTranspose2d(inner_nc * 2,
outer_nc,
kernel_size=4,
stride=2,
padding=1)
down = [downconv]
up = [uprelu, upconv, nn.Tanh()]
model = down + [submodule] + up
elif innermost:
upconv = nn.ConvTranspose2d(inner_nc,
outer_nc,
kernel_size=4,
stride=2,
padding=1,
bias=use_bias)
down = [downrelu, downconv]
up = [uprelu, upconv, upnorm]
model = down + up
else:
upconv = nn.ConvTranspose2d(inner_nc * 2,
outer_nc,
kernel_size=4,
stride=2,
padding=1,
bias=use_bias)
down = [downrelu, downconv, downnorm]
up = [uprelu, upconv, upnorm]
if use_dropout:
model = down + [submodule] + up + [nn.Dropout(0.5)]
else:
model = down + [submodule] + up
self.model = nn.Sequential(*model)
def __init__(self, receptive_field=1024, mgc_size=60, upsample_size=200, filter_size=256,
num_blocks=4):
super(VocoderNetwork, self).__init__()
self.RECEPTIVE_FIELD = receptive_field
self.NUM_NETWORKS = 1
self.MGC_SIZE = mgc_size
self.UPSAMPLE_SIZE = upsample_size
self.NUM_BLOCKS = num_blocks
self.convolutions = torch.nn.ModuleList(
[WaveNet(self.RECEPTIVE_FIELD, mgc_size, filter_size) for ii in range(num_blocks)])
self.conditioning = nn.Sequential(nn.Linear(mgc_size, mgc_size * upsample_size), nn.Tanh())
self.pre_output = torch.nn.ModuleList([nn.Linear(filter_size, 256) for ii in range(num_blocks)])
self.mean_layer = torch.nn.ModuleList([nn.Linear(256, 1) for ii in range(num_blocks)])
self.stdev_layer = torch.nn.ModuleList([nn.Linear(256, 1) for ii in range(num_blocks)])
def __init__(self,
input_dim,
output_dim,
kernel_size,
stride,
padding=0,
conv_padding=0,
norm='none',
activation='relu',
pad_type='zero',
transpose=False):
super(Conv2dBlock, self).__init__()
self.use_bias = True
# initialize padding
if pad_type == 'reflect':
self.pad = nn.ReflectionPad2d(padding)
elif pad_type == 'replicate':
self.pad = nn.ReplicationPad2d(padding)
elif pad_type == 'zero':
self.pad = nn.ZeroPad2d(padding)
elif pad_type == 'none':
self.pad = None
else:
assert 0, "Unsupported padding type: {}".format(pad_type)
# initialize normalization
norm_dim = output_dim
if norm == 'bn':
self.norm = nn.BatchNorm2d(norm_dim)
elif norm == 'in':
self.norm = nn.InstanceNorm2d(norm_dim)
elif norm == 'none':
self.norm = None
else:
assert 0, "Unsupported normalization: {}".format(norm)
# initialize activation
if activation == 'relu':
self.activation = nn.ReLU(inplace=True)
elif activation == 'lrelu':
self.activation = nn.LeakyReLU(0.2, inplace=True)
elif activation == 'prelu':
self.activation = nn.PReLU()
elif activation == 'selu':
self.activation = nn.SELU(inplace=True)
elif activation == 'tanh':
self.activation = nn.Tanh()
elif activation == 'none':
self.activation = None
else:
assert 0, "Unsupported activation: {}".format(activation)
# initialize convolution
if transpose:
self.conv = nn.ConvTranspose2d(input_dim,
output_dim,
kernel_size,
stride,
padding=conv_padding,
output_padding=conv_padding,
bias=self.use_bias)
else:
self.conv = nn.Conv2d(input_dim,
output_dim,
kernel_size,
stride,
padding=conv_padding,
bias=self.use_bias)
def __init__(self,
device,
w2v_weights,
tag_to_itx,
hidden_dim,
drop_rate,
bidirectional=False,
freeze=True,
embedding_norm=6,
c2v_weights=None,
pad_word_length=16,
embedder="none",
more_features=False):
super(LstmCrf, self).__init__()
self.device = device
self.hidden_dim = hidden_dim
self.tagset_size = len(tag_to_itx)
self.embedding_dim = w2v_weights.shape[1]
self.w2v_weights = w2v_weights
self.c2v_weights = c2v_weights
self.pad_word_length = pad_word_length
self.bidirectional = bidirectional
self.embedder = embedder
self.more_features = more_features
self.drop_rate = drop_rate
self.drop = nn.Dropout(self.drop_rate)
# Use the Elmo embedder instead of the classical ones.
if self.embedder != "none":
if self.embedder == "elmo-combined":
self.embedding = ElmoCombiner()
elif self.embedder == "elmo":
self.embedding = ElmoCombiner(freeze=True)
else:
self.embedding = None
self.embedding_dim = 768 if self.embedder == "bert" else 1024
else:
self.embedding = nn.Embedding.from_pretrained(
torch.FloatTensor(w2v_weights), freeze=freeze)
self.embedding.max_norm = embedding_norm
# We add the dimensionality of the other features (POS and spaCy).
if self.more_features:
self.embedding_dim += 58 + 18
# recurrent and mapping to tagset
self.recurrent = nn.LSTM(input_size=self.embedding_dim,
hidden_size=self.hidden_dim //
(1 if not self.bidirectional else 2),
bidirectional=self.bidirectional,
batch_first=True)
self.bnorm = nn.BatchNorm2d(1)
self.fc = nn.Linear(self.hidden_dim, self.tagset_size +
2) # + 2 because of start and end token
self.bnorm2 = nn.BatchNorm2d(1)
# crf for scoring at a global level
self.crf = CRF(self.device, self.tagset_size)
# setup convolution on characters if c2v_weights are passed
if self.c2v_weights is not None:
self.char_embedding_dim = c2v_weights.shape[1]
self.char_embedding = nn.Embedding.from_pretrained(
torch.FloatTensor(c2v_weights), freeze=True)
self.char_embedding.max_norm = embedding_norm
self.feats = 20 # for the output channels of the conv layers
self.recurrent = nn.LSTM(self.embedding_dim + 50,
self.hidden_dim //
(1 if not self.bidirectional else 2),
batch_first=True,
bidirectional=self.bidirectional)
# conv layers for single character, pairs of characters, 3x characters
self.ngram1 = nn.Sequential(
nn.Conv2d(1,
self.feats * 1,
kernel_size=(1, self.char_embedding_dim),
stride=(1, self.char_embedding_dim),
padding=0),
nn.Dropout2d(p=self.drop_rate),
nn.MaxPool2d(kernel_size=(self.pad_word_length, 1)),
nn.Tanh(),
)
self.ngram2 = nn.Sequential(
nn.Conv2d(1,
self.feats * 2,
kernel_size=(2, self.char_embedding_dim),
stride=(1, self.char_embedding_dim),
padding=0),
nn.Dropout2d(p=self.drop_rate),
nn.MaxPool2d(kernel_size=(self.pad_word_length - 1, 1)),
nn.Tanh(),
)
self.ngram3 = nn.Sequential(
nn.Conv2d(1,
self.feats * 3,
kernel_size=(3, self.char_embedding_dim),
stride=(1, self.char_embedding_dim),
padding=0),
nn.Dropout2d(p=self.drop_rate),
nn.MaxPool2d(kernel_size=(self.pad_word_length - 2, 1)),
nn.Tanh(),
)
# seq layers to elaborate on the output of conv layers
self.fc1 = nn.Sequential(nn.Linear(self.feats, 10), )
self.fc2 = nn.Sequential(nn.Linear(self.feats * 2, 20), )
self.fc3 = nn.Sequential(nn.Linear(self.feats * 3, 20), )
def __init__(
self,
input_nc,
output_nc,
ngf=64,
k_size=3,
n_downsampling=8,
norm_layer=nn.BatchNorm2d,
padding_type="reflect",
opt=None,
):
super(GlobalGenerator_DCDCv2, self).__init__()
activation = nn.ReLU(True)
model = [
nn.ReflectionPad2d(3),
nn.Conv2d(input_nc, min(ngf, opt.mc), kernel_size=7, padding=0),
norm_layer(ngf),
activation,
]
### downsample
for i in range(opt.start_r):
mult = 2 ** i
model += [
nn.Conv2d(
min(ngf * mult, opt.mc),
min(ngf * mult * 2, opt.mc),
kernel_size=k_size,
stride=2,
padding=1,
),
norm_layer(min(ngf * mult * 2, opt.mc)),
activation,
]
for i in range(opt.start_r, n_downsampling - 1):
mult = 2 ** i
model += [
nn.Conv2d(
min(ngf * mult, opt.mc),
min(ngf * mult * 2, opt.mc),
kernel_size=k_size,
stride=2,
padding=1,
),
norm_layer(min(ngf * mult * 2, opt.mc)),
activation,
]
model += [
ResnetBlock(
min(ngf * mult * 2, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
model += [
ResnetBlock(
min(ngf * mult * 2, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
mult = 2 ** (n_downsampling - 1)
if opt.spatio_size == 32:
model += [
nn.Conv2d(
min(ngf * mult, opt.mc),
min(ngf * mult * 2, opt.mc),
kernel_size=k_size,
stride=2,
padding=1,
),
norm_layer(min(ngf * mult * 2, opt.mc)),
activation,
]
if opt.spatio_size == 64:
model += [
ResnetBlock(
min(ngf * mult * 2, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
model += [
ResnetBlock(
min(ngf * mult * 2, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
# model += [nn.Conv2d(min(ngf * mult * 2, opt.mc), min(ngf, opt.mc), 1, 1)]
if opt.feat_dim > 0:
model += [nn.Conv2d(min(ngf * mult * 2, opt.mc), opt.feat_dim, 1, 1)]
self.encoder = nn.Sequential(*model)
# decode
model = []
if opt.feat_dim > 0:
model += [nn.Conv2d(opt.feat_dim, min(ngf * mult * 2, opt.mc), 1, 1)]
# model += [nn.Conv2d(min(ngf, opt.mc), min(ngf * mult * 2, opt.mc), 1, 1)]
o_pad = 0 if k_size == 4 else 1
mult = 2 ** n_downsampling
model += [
ResnetBlock(
min(ngf * mult, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
if opt.spatio_size == 32:
model += [
nn.ConvTranspose2d(
min(ngf * mult, opt.mc),
min(int(ngf * mult / 2), opt.mc),
kernel_size=k_size,
stride=2,
padding=1,
output_padding=o_pad,
),
norm_layer(min(int(ngf * mult / 2), opt.mc)),
activation,
]
if opt.spatio_size == 64:
model += [
ResnetBlock(
min(ngf * mult, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
for i in range(1, n_downsampling - opt.start_r):
mult = 2 ** (n_downsampling - i)
model += [
ResnetBlock(
min(ngf * mult, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
model += [
ResnetBlock(
min(ngf * mult, opt.mc),
padding_type=padding_type,
activation=activation,
norm_layer=norm_layer,
opt=opt,
)
]
model += [
nn.ConvTranspose2d(
min(ngf * mult, opt.mc),
min(int(ngf * mult / 2), opt.mc),
kernel_size=k_size,
stride=2,
padding=1,
output_padding=o_pad,
),
norm_layer(min(int(ngf * mult / 2), opt.mc)),
activation,
]
for i in range(n_downsampling - opt.start_r, n_downsampling):
mult = 2 ** (n_downsampling - i)
model += [
nn.ConvTranspose2d(
min(ngf * mult, opt.mc),
min(int(ngf * mult / 2), opt.mc),
kernel_size=k_size,
stride=2,
padding=1,
output_padding=o_pad,
),
norm_layer(min(int(ngf * mult / 2), opt.mc)),
activation,
]
if opt.use_segmentation_model:
model += [nn.ReflectionPad2d(3), nn.Conv2d(min(ngf, opt.mc), output_nc, kernel_size=7, padding=0)]
else:
model += [
nn.ReflectionPad2d(3),
nn.Conv2d(min(ngf, opt.mc), output_nc, kernel_size=7, padding=0),
nn.Tanh(),
]
self.decoder = nn.Sequential(*model)
def __init__(self, channels=3):
super(Generator, self).__init__()
self.down1 = UNetDown(channels, 64, normalize=False)
self.down2 = UNetDown(64, 128)
self.down3 = UNetDown(128, 256)
self.down4 = UNetDown(256, 512, dropout=0.5)
self.down5 = UNetDown(512, 512, dropout=0.5)
self.down6 = UNetDown(512, 512, dropout=0.5)
self.down7 = UNetDown(512, 512, dropout=0.5, normalize=False)
self.up1 = UNetUp(512, 512, dropout=0.5)
self.up2 = UNetUp(1024, 512, dropout=0.5)
self.up3 = UNetUp(1024, 512, dropout=0.5)
self.up4 = UNetUp(1024, 256)
self.up5 = UNetUp(512, 128)
self.up6 = UNetUp(256, 64)
self.final = nn.Sequential(nn.ConvTranspose2d(128, channels, 4, stride=2, padding=1), nn.Tanh())
def __init__(self, input_nc, output_nc, ngf=64, norm_layer=nn.BatchNorm2d, use_dropout=False, n_blocks=6, gpu_ids=[], padding_type='reflect'):
assert(n_blocks >= 0)
super(ResnetGeneratorMM, self).__init__()
self.input_nc = input_nc
self.output_nc = output_nc
self.ngf = ngf
self.gpu_ids = gpu_ids
if type(norm_layer) == functools.partial:
use_bias = norm_layer.func == nn.InstanceNorm2d
else:
use_bias = norm_layer == nn.InstanceNorm2d
model1 = [nn.ReflectionPad2d(3),
nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0,
bias=use_bias),
norm_layer(ngf),
nn.ReLU(True)]
model2 = [nn.ReflectionPad2d(3),
nn.Conv2d(input_nc, ngf, kernel_size=7, padding=0,
bias=use_bias),
norm_layer(ngf),
nn.ReLU(True)]
n_downsampling = 2
for i in range(n_downsampling):
mult = 2**i
model1 += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3,
stride=2, padding=1, bias=use_bias),
norm_layer(ngf * mult * 2),
nn.ReLU(True)]
model2 += [nn.Conv2d(ngf * mult, ngf * mult * 2, kernel_size=3,
stride=2, padding=1, bias=use_bias),
norm_layer(ngf * mult * 2),
nn.ReLU(True)]
pre_f_blocks = 4
pre_l_blocks = 7
mult = 2**n_downsampling
for i in range(pre_f_blocks):
model1 += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
model2 += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
model_pre = []
model_post = []
for i in range(pre_f_blocks,pre_l_blocks):
model_pre += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
for i in range(pre_l_blocks,n_blocks):
model_post += [ResnetBlock(ngf * mult, padding_type=padding_type, norm_layer=norm_layer, use_dropout=use_dropout, use_bias=use_bias)]
model_fusion = []
#p = 0
#if padding_type == 'reflect':
# model_fusion += [nn.ReflectionPad2d(1)]
#elif padding_type == 'replicate':
# model_fusion += [nn.ReplicationPad2d(1)]
#elif padding_type == 'zero':
# p = 1
#else:
# raise NotImplementedError('padding [%s] is not implemented' % padding_type)
model_fusion = [nn.Conv2d(ngf * mult *2, ngf * mult , kernel_size=3, padding=1, bias=use_bias),
norm_layer(ngf * mult),
nn.ReLU(True)]
model = []
for i in range(n_downsampling):
mult = 2**(n_downsampling - i)
model_post += [nn.ConvTranspose2d(ngf * mult, int(ngf * mult / 2),
kernel_size=3, stride=2,
padding=1, output_padding=1,
bias=use_bias),
norm_layer(int(ngf * mult / 2)),
nn.ReLU(True)]
model_post += [nn.ReflectionPad2d(3)]
model_post += [nn.Conv2d(ngf, output_nc, kernel_size=7, padding=0)]
model_post += [nn.Tanh()]
self.model1 = nn.Sequential(*model1)
self.model2 = nn.Sequential(*model2)
self.model_pre = nn.Sequential(*model_pre)
self.model_post = nn.Sequential(*model_post)
self.model_fusion = nn.Sequential(*model_fusion)
def generate(self, trg_input, dec_hidden, enc_context, src_map=None, oov_list=None, max_len=1, return_attention=False):
'''
Given the initial input, state and the source contexts, return the top K restuls for each time step
:param trg_input: just word indexes of target texts (usually zeros indicating BOS <s>)
:param dec_hidden: hidden states for decoder RNN to start with
:param enc_context: context encoding vectors
:param src_map: required if it's copy model
:param oov_list: required if it's copy model
:param k (deprecated): Top K to return
:param feed_all_timesteps: it's one-step predicting or feed all inputs to run through all the time steps
:param get_attention: return attention vectors?
:return:
'''
# assert isinstance(input_list, list) or isinstance(input_list, tuple)
# assert isinstance(input_list[0], list) or isinstance(input_list[0], tuple)
batch_size = trg_input.size(0)
src_len = enc_context.size(1)
trg_len = trg_input.size(1)
context_dim = enc_context.size(2)
trg_hidden_dim = self.trg_hidden_dim
attn_weights = []
copy_weights = []
log_probs = []
# enc_context has to be reshaped before dot attention (batch_size, src_len, context_dim) -> (batch_size, src_len, trg_hidden_dim)
enc_context = nn.Tanh()(self.encoder2decoder_hidden(enc_context.contiguous().view(-1, context_dim))).view(batch_size, src_len, trg_hidden_dim)
for i in range(max_len):
# print('TRG_INPUT: %s' % str(trg_input.size()))
# print(trg_input.data.numpy())
trg_emb = self.embedding(trg_input) # (batch_size, trg_len = 1, emb_dim)
trg_emb = trg_emb.permute(1, 0, 2) # (trg_len, batch_size, embed_dim)
# (seq_len, batch_size, hidden_size * num_directions)
decoder_output, dec_hidden = self.decoder(
trg_emb, dec_hidden
)
# Get the h_tilde (hidden after attention) and attention weights
h_tilde, attn_weight, attn_logit = self.attention_layer(decoder_output.permute(1, 0, 2), enc_context)
# compute the output decode_logit and read-out as probs: p_x = Softmax(W_s * h_tilde)
# (batch_size, trg_len, trg_hidden_size) -> (batch_size, 1, vocab_size)
decoder_logit = self.decoder2vocab(h_tilde.view(-1, trg_hidden_dim))
if not hasattr(self, 'copy_model'):
decoder_log_prob = torch.nn.functional.log_softmax(decoder_logit, dim=-1).view(batch_size, 1, self.vocab_size)
else:
decoder_logit = decoder_logit.view(batch_size, 1, self.vocab_size)
# copy_weights and copy_logits is (batch_size, trg_len, src_len)
if self.copy_attention_layer:
_, copy_weight, copy_logit = self.copy_attention_layer(decoder_output.permute(1, 0, 2), enc_context)
else:
copy_weight = attn_weight
copy_logit = attn_logit
copy_weights.append(copy_weight.permute(1, 0, 2)) # (1, batch_size, src_len)
# merge the generative and copying probs (batch_size, 1, vocab_size + max_unk_word)
decoder_log_prob = self.merge_copy_probs(decoder_logit, copy_logit, src_map, oov_list)
# Prepare for the next iteration, get the top word, top_idx and next_index are (batch_size, K)
top_1_v, top_1_idx = decoder_log_prob.data.topk(1, dim=-1) # (batch_size, 1)
trg_input = Variable(top_1_idx.squeeze(2))
# trg_input = Variable(top_1_idx).cuda() if torch.cuda.is_available() else Variable(top_1_idx) # (batch_size, 1)
# append to return lists
log_probs.append(decoder_log_prob.permute(1, 0, 2)) # (1, batch_size, vocab_size)
attn_weights.append(attn_weight.permute(1, 0, 2)) # (1, batch_size, src_len)
# permute to trg_len first, otherwise the cat operation would mess up things
log_probs = torch.cat(log_probs, 0).permute(1, 0, 2) # (batch_size, max_len, K)
attn_weights = torch.cat(attn_weights, 0).permute(1, 0, 2) # (batch_size, max_len, src_seq_len)
# Only return the hidden vectors of the last time step.
# tuple of (num_layers * num_directions, batch_size, trg_hidden_dim)=(1, batch_size, trg_hidden_dim)
# Return final outputs, hidden states, and attention weights (for visualization)
if return_attention:
if not hasattr(self, 'copy_model'):
return log_probs, dec_hidden, attn_weights
else:
copy_weights = torch.cat(copy_weights, 0).permute(1, 0, 2) # (batch_size, max_len, src_seq_len)
return log_probs, dec_hidden, (attn_weights, copy_weights)
else:
return log_probs, dec_hidden
