def create_model(self, hid_dim, num_encoder_layer, num_decoder_layer, num_head, pf_dim, dropout):
self.encoder = Encoder(self.vocab_size, hid_dim, num_encoder_layer, num_head, pf_dim,
EncoderLayer, SelfAttention, PositionwiseFeedforward, dropout, self.device)
self.decoder = dec = Decoder(self.vocab_size, hid_dim, num_decoder_layer, num_head, pf_dim,
DecoderLayer, SelfAttention, PositionwiseFeedforward, dropout, self.device)
self.model = Seq2Seq(self.encoder, self.decoder, self.pad_idx, self.device).to(self.device)
return self.model
def make_model(src_vocab,
tgt_vocab,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1):
attn = MultiHeadedAttention(h, d_model, dropout)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
pe = PositionalEncoding(d_model, dropout)
encoder_layer = EncoderLayer(d_model, copy(attn), copy(ff), dropout)
encoder = Encoder(encoder_layer, N)
decoder_layer = DecoderLayer(d_model, copy(attn), copy(attn), copy(ff), dropout)
decoder = Decoder(decoder_layer, N)
src_embed = nn.Sequential(Embedding(src_vocab, d_model), copy(pe))
tgt_embed = nn.Sequential(Embedding(tgt_vocab, d_model), copy(pe))
generator = Generator(d_model, tgt_vocab)
model = EncoderDecoder(encoder, decoder, src_embed, tgt_embed, generator)
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform(p)
return model
def __init__(self,
x_dim,
y_dim,
mlp_hidden_size_list,
latent_dim,
use_rnn=True,
use_self_attention=True,
le_self_attention_type="dot",
de_self_attention_type="dot",
de_cross_attention_type="multihead",
use_deter_path=True,
**kwargs):
super(ANP_RNN_Model, self).__init__()
self.x_dim = x_dim
self.y_dim = y_dim
self.mlp_hidden_size_list = mlp_hidden_size_list
self.latent_dim = latent_dim
self.use_rnn = use_rnn
self.use_deter_path = use_deter_path
if self.use_rnn:
self.x_dim = latent_dim
self.num_rnn_layers = 2
self.rnn = nn.LSTM(x_dim, self.x_dim, self.num_rnn_layers)
else:
self.rnn_hidden_size = None
self.num_rnn_layers = None
self.rnn = None
# NOTICE: Latent Encoder
self._latent_encoder = LatentEncoder(
input_x_dim=self.x_dim,
input_y_dim=self.y_dim,
hidden_dim_list=self.mlp_hidden_size_list,
latent_dim=self.latent_dim,
use_self_attn=use_self_attention,
self_attention_type=le_self_attention_type)
# NOTICE : Decoder
self._decoder = Decoder(
x_dim=self.x_dim,
y_dim=self.y_dim,
mid_hidden_dim_list=self.mlp_hidden_size_list,
latent_dim=self.latent_dim, # the dim of last axis of sc and z..
use_deterministic_path=
True, # whether use d_path or not will change the size of input
use_lstm=False)
# NOTICE: Deterministic Encoder
self._deter_encoder = DeterministicEncoder(
input_x_dim=self.x_dim,
input_y_dim=self.y_dim,
hidden_dim_list=self.mlp_hidden_size_list,
latent_dim=self.latent_dim, # the dim of last axis of r..
self_attention_type=de_self_attention_type,
use_self_attn=use_self_attention,
attention_layers=2,
use_lstm=False,
cross_attention_type=de_cross_attention_type,
attention_dropout=0)
def create_model(vocab_src, vocab_tgt, config):
inference_model = InferenceModel(config)
encoder = Encoder(config)
attention = create_attention(config)
decoder = Decoder(attention, vocab_tgt.size(), config)
language_model = LanguageModel(vocab_src.size(), config)
model = AEVNMT(vocab_src, vocab_tgt, inference_model, encoder, decoder,
language_model, config)
return model
def create_model(src_vocab_size, trg_vocab_size, hid_dim, num_encoder_layer,
num_decoder_layer, num_head, pf_dim, dropout, pad_idx,
device):
encoder = Encoder(src_vocab_size, hid_dim, num_encoder_layer, num_head,
pf_dim, EncoderLayer, SelfAttention,
PositionwiseFeedforward, dropout, device)
decoder = dec = Decoder(trg_vocab_size, hid_dim, num_decoder_layer,
num_head, pf_dim, DecoderLayer, SelfAttention,
PositionwiseFeedforward, dropout, device)
model = Seq2Seq(encoder, decoder, pad_idx, device).to(device)
return model
def __init__(self, vocabulary, training):
super(Model, self).__init__()
self._training = training
self._embedding = Embedding(vocabulary=vocabulary)
self._encoder = Encoder(embedding=self._embedding, training=training)
self._decoder = Decoder(embedding=self._embedding, training=training)
if training:
self._teacher_forcing = TeacherForcing()
# TODO: Look at other possible loss functions
self._loss = masked_nll_loss
def __init__(self, embedding_dim, num_word_embeddings, num_char_embeddings,
kernels, num_input_channels, num_output_channels,
rnn_hidden_dim, hidden_dim, output_dim, num_layers,
bidirectional, dropout_p, word_padding_idx, char_padding_idx):
super(NewsModel, self).__init__()
self.encoder = Encoder(embedding_dim, num_word_embeddings,
num_char_embeddings, kernels,
num_input_channels, num_output_channels,
rnn_hidden_dim, num_layers, bidirectional,
word_padding_idx, char_padding_idx)
self.decoder = Decoder(rnn_hidden_dim, hidden_dim, output_dim,
dropout_p)
def __init__(self,
num_layers=4,
d_model=512,
num_heads=8,
dff=2048,
pe_max_len=8000,
target_vocab_size=8000,
rate=0.1,
config=None,
logger=None):
super(Transformer, self).__init__()
if config is not None:
num_enc_layers = config.model.N_encoder
if logger is not None:
logger.info('config.model.N_encoder: ' + str(num_enc_layers))
num_dec_layers = config.model.N_decoder
if logger is not None:
logger.info('config.model.N_decoder: ' + str(num_dec_layers))
d_model = config.model.d_model
if logger is not None:
logger.info('config.model.d_model: ' + str(d_model))
num_heads = config.model.n_heads
if logger is not None:
logger.info('config.model.n_heads: ' + str(num_heads))
dff = config.model.d_ff
if logger is not None:
logger.info('config.model.d_ff: ' + str(dff))
pe_max_len = config.model.pe_max_len
if logger is not None:
logger.info('config.model.pe_max_len:' + str(pe_max_len))
target_vocab_size = config.model.vocab_size
if logger is not None:
logger.info('config.model.vocab_size:' +
str(target_vocab_size))
rate = config.model.dropout
if logger is not None:
logger.info('config.model.dropout: ' + str(rate))
else:
print('use default params')
num_enc_layers = num_layers
num_dec_layers = num_layers
self.encoder = Encoder(num_enc_layers, d_model, num_heads, dff,
pe_max_len, 'encoder', rate)
self.decoder = Decoder(num_dec_layers, d_model, num_heads, dff,
target_vocab_size, 'decoder', pe_max_len, rate)
def __init__(self,
x_dim,
y_dim,
mlp_hidden_size_list,
latent_dim,
use_rnn=False,
use_self_attention=True,
use_deter_path=True,
self_attention_type="dot",
cross_attention_type="multihead",
**kwargs):
super(NP_Model, self).__init__()
self.x_dim = x_dim
self.y_dim = y_dim
self.mlp_hidden_size_list = mlp_hidden_size_list
self.latent_dim = latent_dim
self.use_rnn = use_rnn
self.use_self_attention = use_self_attention
self.use_deter_path = use_deter_path
self.self_attention_type = self_attention_type
self.cross_attention_type = cross_attention_type
# NOTICE: Latent Encoder
self._latent_encoder = LatentEncoder(input_x_dim=self.x_dim,
input_y_dim=self.y_dim,
hidden_dim_list=self.mlp_hidden_size_list,
latent_dim=self.latent_dim)
# NOTICE : Decoder
self._decoder = Decoder(x_dim=self.x_dim,
y_dim=self.y_dim,
mid_hidden_dim_list=self.mlp_hidden_size_list,
latent_dim=self.latent_dim, # the dim of last axis of sc and z..
use_deterministic_path=self.use_deter_path, # whether use d_path or not will change the size of input
use_lstm=False)
# NOTICE: Deterministic Encoder
self._deter_encoder = DeterministicEncoder(input_x_dim=self.x_dim,
input_y_dim=self.y_dim,
hidden_dim_list=self.mlp_hidden_size_list,
latent_dim=self.latent_dim, # the dim of last axis of r..
self_attention_type=self.self_attention_type,
use_self_attn=self.use_self_attention,
attention_layers=2,
use_lstm=False,
cross_attention_type='uniform',
cross_attention_rep='identity',
attention_dropout=0)
def __init__(self, cfg):
super(MultiInstanceRecognition, self).__init__()
self.cfg = cfg
self.backbone = ResNet50(pretrained='torchvision://resnet50')
self.neck = FPN(cfg['feature_channels'], cfg['fpn_out_channels'],
LastLevelMaxPool())
self.roialign = MultiInstanceAlign(cfg['num_instances'],
cfg['roi_size'],
cfg['roi_feature_steps'])
self.encoder = CnnEncoder(
cfg['fpn_out_channels'] * cfg['num_instances'],
cfg['encoder_channels'])
self.voc_len = cfg['voc_len']
self.decoder = Decoder(self.cfg)
self.num_instances = cfg['num_instances']
def __init__(self,
num_layers,
d_model,
num_heads,
dff,
input_vocab_size,
target_vocab_size,
pe_input,
pe_target,
rate=config.dropout_rate):
super(Transformer, self).__init__()
self.encoder = Encoder(num_layers, d_model, num_heads, dff,
input_vocab_size, pe_input, rate)
self.decoder = Decoder(num_layers, d_model, num_heads, dff,
target_vocab_size, pe_target, rate)
self.final_layer = tf.keras.layers.Dense(target_vocab_size)
def __init__(self,conv_in=1,conv_out=256,conv_kernel=9,pri_cap_num=8,pri_out=256,pri_kernel=3,
pri_stride=2,digit_in=8,digit_out=16,num_classes=26,num_routes=256*2*2,share_weights=False,
decoder_in=16*10,de_outputI=512,de_outputII=1024,de_outputIII=1200,device=torch.device('cuda')):
super(CapsNet, self).__init__()
decoder_in = decoder_in // 10 * num_classes
# print(decoder_in)
self.device = device
# self.convLayer = ConvLayer(conv_in,conv_out,conv_kernel)
self.alexNetLayer = AlexNet(num_classes)
self.primary_capsLayer = PrimaryCaps(pri_cap_num,conv_out,pri_out,pri_kernel,pri_stride)
self.digit_capsLayer = DigitCaps(digit_in,digit_out,num_classes,num_routes,share_weights)
self.decoder = Decoder(decoder_in,de_outputI,de_outputII,de_outputIII,num_classes)
self.dropout = nn.Dropout(p=0.3)
self.batch_norm = nn.BatchNorm2d(num_features=3)
self.relu = nn.ReLU() # for computing loss
self.mse_loss = nn.MSELoss()
if __name__ == "__main__":
# Directorio actual
current_dir = os.path.realpath(os.path.dirname(__file__))
# Directorio donde se almacenarán los ficheros descargados y generados
files_path = current_dir + '\\files'
if not os.path.exists(files_path):
os.mkdir(files_path)
start = perf_counter()
# Descarga del fichero
xml_path, dtd_path = download_file(files_path)
decoded_xml = files_path + '\\decoded-dblp.xml'
# Decodificación del fichero
decoder = Decoder(xml_path, decoded_xml, dtd_path)
decoder.recode_file()
# Obtención de los identificadores del fichero (España)
scrapper = Scrapper()
ids = scrapper.scrape(decoded_xml, mask='spain')
# Descarga de datos de los autores
crawler = Crawler()
authors_data = {}
pbar = tqdm(total=len(ids), desc="Descargando datos de los investigadores")
for id in ids:
authors, props = crawler.crawl(id)
# Si al descargar los datos de un autor el servidor devuelve HTTP404 (Not found) se descarta ese autor
if props is not None:
def create_model(vocab_src, vocab_tgt, config):
encoder = Encoder(config)
attention = create_attention(config)
decoder = Decoder(attention, vocab_tgt.size(), config)
model = CondNMT(vocab_src, vocab_tgt, encoder, decoder, config)
return model
def __init__(self, input_channels, h_size, z_size):
super(VAE, self).__init__()
self.encoder = Encoder(input_channels, h_size, z_size)
self.decoder = Decoder(input_channels, z_size)
class ModelGraph(object):
def __init__(self,
word_vocab=None,
POS_vocab=None,
NER_vocab=None,
flags=None,
mode='ce_train'):
# mode can have the following values:
# 'ce_train',
# 'rl_train',
# 'evaluate',
# 'evaluate_bleu',
# 'decode'.
# it is different from mode in decoder which can be
# 'ce_train', 'loss', 'greedy' or 'sample'
self.mode = mode
# is_training controls whether to use dropout
is_training = True if mode in ('ce_train') else False
self.flags = flags
self.word_vocab = word_vocab
# create placeholders
self.create_placeholders()
# create encoder
self.encoder = Encoder(flags,
self.passage_words,
self.passage_POSs,
self.passage_NERs,
self.passage_lengths,
self.answer_span,
word_vocab=word_vocab,
POS_vocab=POS_vocab,
NER_vocab=NER_vocab)
# encode the input instance
self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.decoder_init_state = self.encoder.encode(
is_training=is_training)
max_passage_length = tf.shape(self.passage_words)[1]
self.passage_mask = tf.sequence_mask(self.passage_lengths,
max_passage_length,
dtype=tf.float32)
loss_weights = tf.sequence_mask(
self.question_lengths, flags.max_question_len,
dtype=tf.float32) # [batch_size, gen_steps]
loss_weights_rl = tf.sequence_mask(
self.question_lengths_rl, flags.max_question_len,
dtype=tf.float32) # [batch_size, gen_steps]
with tf.variable_scope("generator"):
# create decoder
self.decoder = Decoder(flags, word_vocab, self.rewards,
is_training)
if mode == 'decode':
self.context_t_1 = tf.placeholder(
tf.float32, [None, self.encoder_dim],
name='context_t_1') # [batch_size, encoder_dim]
self.coverage_t_1 = tf.placeholder(
tf.float32, [None, None],
name='coverage_t_1') # [batch_size, encoder_dim]
self.word_t = tf.placeholder(tf.int32, [None],
name='word_t') # [batch_size]
(self.state_t, self.context_t, self.coverage_t,
self.attn_dist_t, self.p_gen_t, self.ouput_t,
self.topk_log_probs, self.topk_ids, self.greedy_prediction,
self.multinomial_prediction) = self.decoder.decode(
self.decoder_init_state, self.context_t_1,
self.coverage_t_1, self.word_t,
self.encoder_hidden_states, self.encoder_features,
self.passage_words, self.passage_mask)
# not buiding training op for this mode
return
elif mode == 'evaluate_bleu':
_, _, self.greedy_words = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='greedy')
# not buiding training op for this mode
return
elif mode in ('ce_train', 'evaluate'):
self.accu, self.loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='ce_train')
if mode == 'evaluate':
# not buiding training op for evaluation
return
elif mode == 'rl_train':
_, self.loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='loss')
tf.get_variable_scope().reuse_variables()
_, _, self.greedy_words = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
None,
mode='greedy')
elif mode == 'rl_ce_train':
self.accu, self.ce_loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='ce_train')
tf.get_variable_scope().reuse_variables()
_, self.rl_loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs_rl,
self.question_words_rl,
loss_weights_rl,
mode='loss')
self.loss = BETA * self.ce_loss + self.rl_loss
_, _, self.greedy_words = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
None,
mode='greedy')
# defining optimizer and train op
optimizer = tf.train.AdagradOptimizer(
learning_rate=flags.learning_rate)
tvars = tf.trainable_variables()
total_parameters = 0
for variable in tvars:
shape = variable.get_shape()
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
total_parameters += variable_parameters
print("Total number of parameters is equal: %s" % total_parameters)
if flags.lambda_l2 > 0.0:
l2_loss = tf.add_n(
[tf.nn.l2_loss(v) for v in tvars if v.get_shape().ndims > 1])
self.loss = self.loss + flags.lambda_l2 * l2_loss
grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars),
flags.clip_value)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
ema = tf.train.ExponentialMovingAverage(decay=0.9999)
with tf.control_dependencies([self.train_op]):
self.train_op = ema.apply(tvars)
with tf.variable_scope('backup_variables'):
backup_vars = [
tf.get_variable(var.op.name,
dtype=var.value().dtype,
trainable=False,
initializer=var.initialized_value())
for var in tvars
]
save_backup_vars_op = tf.group(
*(tf.assign(bck, var.read_value())
for var, bck in zip(tvars, backup_vars)))
with tf.control_dependencies([save_backup_vars_op]):
self.ema_to_vars_op = tf.group(
*(tf.assign(var,
ema.average(var).read_value()) for var in tvars))
self.restore_backup_vars_op = tf.group(
*(tf.assign(var, bck.read_value())
for var, bck in zip(tvars, backup_vars)))
def create_placeholders(self):
# build placeholder for input passage/article
self.passage_lengths = tf.placeholder(tf.int32, [None],
name='passage_lengths')
self.passage_words = tf.placeholder(
tf.int32, [None, None],
name="passage_words") # [batch_size, passage_len]
self.passage_POSs = tf.placeholder(
tf.int32, [None, None],
name="passage_POSs") # [batch_size, passage_len]
self.passage_NERs = tf.placeholder(
tf.int32, [None, None],
name="passage_NERs") # [batch_size, passage_len]
# build placeholder for answer
self.answer_span = tf.placeholder(tf.float32, [None, None],
name="answer_span") # [batch_size]
# build placeholder for question
self.decoder_inputs = tf.placeholder(
tf.int32, [None, self.flags.max_question_len],
name="decoder_inputs") # [batch_size, gen_steps]
self.question_words = tf.placeholder(
tf.int32, [None, self.flags.max_question_len],
name="question_words") # [batch_size, gen_steps]
self.question_lengths = tf.placeholder(
tf.int32, [None], name="question_lengths") # [batch_size]
self.decoder_inputs_rl = tf.placeholder(
tf.int32, [None, self.flags.max_question_len],
name="decoder_inputs_rl") # [batch_size, gen_steps]
self.question_words_rl = tf.placeholder(
tf.int32, [None, self.flags.max_question_len],
name="question_words_rl") # [batch_size, gen_steps]
self.question_lengths_rl = tf.placeholder(
tf.int32, [None], name="question_lengths_rl") # [batch_size]
# build placeholder for reinforcement learning
self.rewards = tf.placeholder(tf.float32, [None], name="rewards")
def run_greedy(self, sess, batch):
feed_dict = self.run_encoder(sess, batch, only_feed_dict=True)
feed_dict[self.decoder_inputs] = batch.decoder_inputs
return sess.run(self.greedy_words, feed_dict)
def ce_train(self, sess, batch, only_eval=False):
feed_dict = self.run_encoder(sess, batch, only_feed_dict=True)
feed_dict[self.decoder_inputs] = batch.decoder_inputs
feed_dict[self.question_words] = batch.question_words
feed_dict[self.question_lengths] = batch.question_lengths
if only_eval:
return sess.run([self.accu, self.loss], feed_dict)
else:
return sess.run([self.train_op, self.loss], feed_dict)[1]
def rl_train(self, sess, batch, with_ce):
feed_dict = self.run_encoder(sess, batch, only_feed_dict=True)
feed_dict[self.decoder_inputs] = batch.decoder_inputs
greedy_outputs = sess.run(self.greedy_words, feed_dict)
greedy_outputs = greedy_outputs.tolist()
gold_output = batch.question_words.tolist()
# baseline outputs by flipping coin
flipp = 0.1
baseline_outputs = np.copy(batch.question_words)
for i in range(batch.question_words.shape[0]):
seq_len = min(self.flags.max_question_len,
batch.question_lengths[i] -
1) # don't change stop token '</s>'
for j in range(seq_len):
if greedy_outputs[i][j] != 0 and random.random() < flipp:
baseline_outputs[i, j] = greedy_outputs[i][j]
baseline_outputs = baseline_outputs.tolist()
rl_inputs = []
rl_outputs = []
rl_input_lengths = []
rewards = []
for i, (baseline_output, greedy_output) in enumerate(
zip(baseline_outputs, greedy_outputs)):
_, baseline_output_words = self.word_vocab.getLexical(
baseline_output)
greedy_output, greedy_output_words = self.word_vocab.getLexical(
greedy_output)
_, gold_output_words = self.word_vocab.getLexical(gold_output[i])
rl_inputs.append([int(batch.decoder_inputs[i, 0])] +
greedy_output[:-1])
rl_outputs.append(greedy_output)
rl_input_lengths.append(len(greedy_output))
baseline_output_words_list = baseline_output_words.split()
greedy_output_words_list = greedy_output_words.split()
gold_output_words_list = gold_output_words.split()
if self.flags.reward_type == 'bleu':
cc = SmoothingFunction()
reward = sentence_bleu([gold_output_words_list],
greedy_output_words_list,
smoothing_function=cc.method3)
baseline = sentence_bleu([gold_output_words_list],
baseline_output_words_list,
smoothing_function=cc.method3)
rewards.append(reward - baseline)
elif self.flags.reward_type == 'rouge':
reward = rouge.rouge([gold_output_words],
[greedy_output_words])["rouge_l/f_score"]
baseline = rouge.rouge(
[gold_output_words],
[baseline_output_words])["rouge_l/f_score"]
rewards.append(reward - baseline)
else:
raise ValueError("Reward type is not bleu or rouge!")
rl_inputs = padding_utils.pad_2d_vals(rl_inputs, len(rl_inputs),
self.flags.max_question_len)
rl_outputs = padding_utils.pad_2d_vals(rl_outputs, len(rl_outputs),
self.flags.max_question_len)
rl_input_lengths = np.array(rl_input_lengths, dtype=np.int32)
rewards = np.array(rewards, dtype=np.float32)
#reward = rescale(reward)
assert rl_inputs.shape == rl_outputs.shape
feed_dict = self.run_encoder(sess, batch, only_feed_dict=True)
feed_dict[self.rewards] = rewards
if with_ce:
feed_dict[self.decoder_inputs_rl] = rl_inputs
feed_dict[self.question_words_rl] = rl_outputs
feed_dict[self.question_lengths_rl] = rl_input_lengths
feed_dict[self.decoder_inputs] = batch.decoder_inputs
feed_dict[self.question_words] = batch.question_words
feed_dict[self.question_lengths] = batch.question_lengths
else:
feed_dict[self.decoder_inputs] = rl_inputs
feed_dict[self.question_words] = rl_outputs
feed_dict[self.question_lengths] = rl_input_lengths
_, loss = sess.run([self.train_op, self.loss], feed_dict)
return loss
def run_encoder(self, sess, batch, only_feed_dict=False):
feed_dict = {}
feed_dict[self.passage_lengths] = batch.sent1_length
if self.flags.with_word:
feed_dict[self.passage_words] = batch.sent1_word
if self.flags.with_POS:
feed_dict[self.passage_POSs] = batch.sent1_POS
if self.flags.with_NER:
feed_dict[self.passage_NERs] = batch.sent1_NER
if self.flags.with_answer_span:
feed_dict[self.answer_span] = batch.answer_span
if only_feed_dict:
return feed_dict
return sess.run([
self.encoder_hidden_states, self.decoder_init_state,
self.encoder_features, self.passage_words, self.passage_mask
], feed_dict)
def __init__(self,
word_vocab=None,
POS_vocab=None,
NER_vocab=None,
flags=None,
mode='ce_train'):
# mode can have the following values:
# 'ce_train',
# 'rl_train',
# 'evaluate',
# 'evaluate_bleu',
# 'decode'.
# it is different from mode in decoder which can be
# 'ce_train', 'loss', 'greedy' or 'sample'
self.mode = mode
# is_training controls whether to use dropout
is_training = True if mode in ('ce_train') else False
self.flags = flags
self.word_vocab = word_vocab
# create placeholders
self.create_placeholders()
# create encoder
self.encoder = Encoder(flags,
self.passage_words,
self.passage_POSs,
self.passage_NERs,
self.passage_lengths,
self.answer_span,
word_vocab=word_vocab,
POS_vocab=POS_vocab,
NER_vocab=NER_vocab)
# encode the input instance
self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.decoder_init_state = self.encoder.encode(
is_training=is_training)
max_passage_length = tf.shape(self.passage_words)[1]
self.passage_mask = tf.sequence_mask(self.passage_lengths,
max_passage_length,
dtype=tf.float32)
loss_weights = tf.sequence_mask(
self.question_lengths, flags.max_question_len,
dtype=tf.float32) # [batch_size, gen_steps]
loss_weights_rl = tf.sequence_mask(
self.question_lengths_rl, flags.max_question_len,
dtype=tf.float32) # [batch_size, gen_steps]
with tf.variable_scope("generator"):
# create decoder
self.decoder = Decoder(flags, word_vocab, self.rewards,
is_training)
if mode == 'decode':
self.context_t_1 = tf.placeholder(
tf.float32, [None, self.encoder_dim],
name='context_t_1') # [batch_size, encoder_dim]
self.coverage_t_1 = tf.placeholder(
tf.float32, [None, None],
name='coverage_t_1') # [batch_size, encoder_dim]
self.word_t = tf.placeholder(tf.int32, [None],
name='word_t') # [batch_size]
(self.state_t, self.context_t, self.coverage_t,
self.attn_dist_t, self.p_gen_t, self.ouput_t,
self.topk_log_probs, self.topk_ids, self.greedy_prediction,
self.multinomial_prediction) = self.decoder.decode(
self.decoder_init_state, self.context_t_1,
self.coverage_t_1, self.word_t,
self.encoder_hidden_states, self.encoder_features,
self.passage_words, self.passage_mask)
# not buiding training op for this mode
return
elif mode == 'evaluate_bleu':
_, _, self.greedy_words = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='greedy')
# not buiding training op for this mode
return
elif mode in ('ce_train', 'evaluate'):
self.accu, self.loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='ce_train')
if mode == 'evaluate':
# not buiding training op for evaluation
return
elif mode == 'rl_train':
_, self.loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='loss')
tf.get_variable_scope().reuse_variables()
_, _, self.greedy_words = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
None,
mode='greedy')
elif mode == 'rl_ce_train':
self.accu, self.ce_loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
loss_weights,
mode='ce_train')
tf.get_variable_scope().reuse_variables()
_, self.rl_loss, _ = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs_rl,
self.question_words_rl,
loss_weights_rl,
mode='loss')
self.loss = BETA * self.ce_loss + self.rl_loss
_, _, self.greedy_words = self.decoder.train(
self.encoder_dim,
self.encoder_hidden_states,
self.encoder_features,
self.passage_words,
self.passage_mask,
self.decoder_init_state,
self.decoder_inputs,
self.question_words,
None,
mode='greedy')
# defining optimizer and train op
optimizer = tf.train.AdagradOptimizer(
learning_rate=flags.learning_rate)
tvars = tf.trainable_variables()
total_parameters = 0
for variable in tvars:
shape = variable.get_shape()
variable_parameters = 1
for dim in shape:
variable_parameters *= dim.value
total_parameters += variable_parameters
print("Total number of parameters is equal: %s" % total_parameters)
if flags.lambda_l2 > 0.0:
l2_loss = tf.add_n(
[tf.nn.l2_loss(v) for v in tvars if v.get_shape().ndims > 1])
self.loss = self.loss + flags.lambda_l2 * l2_loss
grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars),
flags.clip_value)
self.train_op = optimizer.apply_gradients(zip(grads, tvars))
ema = tf.train.ExponentialMovingAverage(decay=0.9999)
with tf.control_dependencies([self.train_op]):
self.train_op = ema.apply(tvars)
with tf.variable_scope('backup_variables'):
backup_vars = [
tf.get_variable(var.op.name,
dtype=var.value().dtype,
trainable=False,
initializer=var.initialized_value())
for var in tvars
]
save_backup_vars_op = tf.group(
*(tf.assign(bck, var.read_value())
for var, bck in zip(tvars, backup_vars)))
with tf.control_dependencies([save_backup_vars_op]):
self.ema_to_vars_op = tf.group(
*(tf.assign(var,
ema.average(var).read_value()) for var in tvars))
self.restore_backup_vars_op = tf.group(
*(tf.assign(var, bck.read_value())
for var, bck in zip(tvars, backup_vars)))
means = config["means"]
std = config["stds"]
channels = config["channel"]
alpha = config["alpha"]
csv_path = config["csv_path"]
img_dir = config["image_dir"]
output_dir = config["output_dir"]
if not os.path.exists(output_dir):
os.mkdir(output_dir)
data = pd.read_csv(csv_path)
paths = data["ImageId"].values
paths = [os.path.join(img_dir, p) for p in paths]
labels = data["TrueLabel"].values
encoder = Encoder(channels, out_ch=2048)
decoder = Decoder(2048, channels)
encoder.load_state_dict(torch.load(config["encoder"], map_location="cpu"))
decoder.load_state_dict(torch.load(config["decoder"], map_location="cpu"))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
encoder.to(device)
decoder.to(device)
encoder.eval()
decoder.eval()
x_adv = []
with torch.no_grad():
bar = tqdm.tqdm(paths)
for path in bar:
filename = os.path.basename(path)
def __init__(self):
super().__init__()
self.encoder = Encoder()
self.decoder = Decoder()
self.egm = EdgeGuidanceModule()
self.wam = WeightedAggregationModule()
# tsf.ToTensor(),
# tsf.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5))
# ])
# train_dataset = CIFAR10("./data", train=True, transform=train_transform, download=True)
# test_dataset = CIFAR10("./data", train=False, transform=val_transform, download=False)
#train_paths, val_paths, train_labels, val_labels = train_test_split(paths, labels, random_state=0, stratify=labels)
train_paths, train_labels = get_paths("./cat_dog/train", img_suffix=".jpg")
val_paths, val_labels = get_paths("./cat_dog/val", img_suffix='.jpg')
train_dataset = PALMClassifyDataset(train_paths, train_labels, augmentation=True, img_size=img_size)
test_dataset = PALMClassifyDataset(val_paths, val_labels, augmentation=False, img_size=img_size)
train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers)
test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers)
encoder = Encoder(in_ch=channels, out_ch=2048)
decoder = Decoder(in_ch=2048, out_ch=channels)
classifier = ResNet(channels, n_layers=50, num_classes=num_classes)
#classifier = resnet18(pretrained=False, num_classes=num_classes, zero_init_residual=False)
# classifier.load_state_dict(torch.load("./best_classifier.pth", map_location="cpu"))
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
encoder.to(device)
decoder.to(device)
classifier.to(device)
encoder_opt = opt.Adam(encoder.parameters(), lr=init_lr, weight_decay=5e-4)
decoder_opt = opt.Adam(decoder.parameters(), lr=init_lr, weight_decay=5e-4)
classifier_opt = opt.Adam(classifier.parameters(), lr=init_lr, weight_decay=5e-4)
c_loss = nn.CrossEntropyLoss()
