def create_model(self, hid_dim, num_encoder_layer, num_decoder_layer, num_head, pf_dim, dropout): self.encoder = Encoder(self.vocab_size, hid_dim, num_encoder_layer, num_head, pf_dim, EncoderLayer, SelfAttention, PositionwiseFeedforward, dropout, self.device) self.decoder = dec = Decoder(self.vocab_size, hid_dim, num_decoder_layer, num_head, pf_dim, DecoderLayer, SelfAttention, PositionwiseFeedforward, dropout, self.device) self.model = Seq2Seq(self.encoder, self.decoder, self.pad_idx, self.device).to(self.device) return self.model
def make_model(src_vocab, tgt_vocab, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1): attn = MultiHeadedAttention(h, d_model, dropout) ff = PositionwiseFeedForward(d_model, d_ff, dropout) pe = PositionalEncoding(d_model, dropout) encoder_layer = EncoderLayer(d_model, copy(attn), copy(ff), dropout) encoder = Encoder(encoder_layer, N) decoder_layer = DecoderLayer(d_model, copy(attn), copy(attn), copy(ff), dropout) decoder = Decoder(decoder_layer, N) src_embed = nn.Sequential(Embedding(src_vocab, d_model), copy(pe)) tgt_embed = nn.Sequential(Embedding(tgt_vocab, d_model), copy(pe)) generator = Generator(d_model, tgt_vocab) model = EncoderDecoder(encoder, decoder, src_embed, tgt_embed, generator) for p in model.parameters(): if p.dim() > 1: nn.init.xavier_uniform(p) return model
def __init__(self, x_dim, y_dim, mlp_hidden_size_list, latent_dim, use_rnn=True, use_self_attention=True, le_self_attention_type="dot", de_self_attention_type="dot", de_cross_attention_type="multihead", use_deter_path=True, **kwargs): super(ANP_RNN_Model, self).__init__() self.x_dim = x_dim self.y_dim = y_dim self.mlp_hidden_size_list = mlp_hidden_size_list self.latent_dim = latent_dim self.use_rnn = use_rnn self.use_deter_path = use_deter_path if self.use_rnn: self.x_dim = latent_dim self.num_rnn_layers = 2 self.rnn = nn.LSTM(x_dim, self.x_dim, self.num_rnn_layers) else: self.rnn_hidden_size = None self.num_rnn_layers = None self.rnn = None # NOTICE: Latent Encoder self._latent_encoder = LatentEncoder( input_x_dim=self.x_dim, input_y_dim=self.y_dim, hidden_dim_list=self.mlp_hidden_size_list, latent_dim=self.latent_dim, use_self_attn=use_self_attention, self_attention_type=le_self_attention_type) # NOTICE : Decoder self._decoder = Decoder( x_dim=self.x_dim, y_dim=self.y_dim, mid_hidden_dim_list=self.mlp_hidden_size_list, latent_dim=self.latent_dim, # the dim of last axis of sc and z.. use_deterministic_path= True, # whether use d_path or not will change the size of input use_lstm=False) # NOTICE: Deterministic Encoder self._deter_encoder = DeterministicEncoder( input_x_dim=self.x_dim, input_y_dim=self.y_dim, hidden_dim_list=self.mlp_hidden_size_list, latent_dim=self.latent_dim, # the dim of last axis of r.. self_attention_type=de_self_attention_type, use_self_attn=use_self_attention, attention_layers=2, use_lstm=False, cross_attention_type=de_cross_attention_type, attention_dropout=0)
def create_model(vocab_src, vocab_tgt, config): inference_model = InferenceModel(config) encoder = Encoder(config) attention = create_attention(config) decoder = Decoder(attention, vocab_tgt.size(), config) language_model = LanguageModel(vocab_src.size(), config) model = AEVNMT(vocab_src, vocab_tgt, inference_model, encoder, decoder, language_model, config) return model
def create_model(src_vocab_size, trg_vocab_size, hid_dim, num_encoder_layer, num_decoder_layer, num_head, pf_dim, dropout, pad_idx, device): encoder = Encoder(src_vocab_size, hid_dim, num_encoder_layer, num_head, pf_dim, EncoderLayer, SelfAttention, PositionwiseFeedforward, dropout, device) decoder = dec = Decoder(trg_vocab_size, hid_dim, num_decoder_layer, num_head, pf_dim, DecoderLayer, SelfAttention, PositionwiseFeedforward, dropout, device) model = Seq2Seq(encoder, decoder, pad_idx, device).to(device) return model
def __init__(self, vocabulary, training): super(Model, self).__init__() self._training = training self._embedding = Embedding(vocabulary=vocabulary) self._encoder = Encoder(embedding=self._embedding, training=training) self._decoder = Decoder(embedding=self._embedding, training=training) if training: self._teacher_forcing = TeacherForcing() # TODO: Look at other possible loss functions self._loss = masked_nll_loss
def __init__(self, embedding_dim, num_word_embeddings, num_char_embeddings, kernels, num_input_channels, num_output_channels, rnn_hidden_dim, hidden_dim, output_dim, num_layers, bidirectional, dropout_p, word_padding_idx, char_padding_idx): super(NewsModel, self).__init__() self.encoder = Encoder(embedding_dim, num_word_embeddings, num_char_embeddings, kernels, num_input_channels, num_output_channels, rnn_hidden_dim, num_layers, bidirectional, word_padding_idx, char_padding_idx) self.decoder = Decoder(rnn_hidden_dim, hidden_dim, output_dim, dropout_p)
def __init__(self, num_layers=4, d_model=512, num_heads=8, dff=2048, pe_max_len=8000, target_vocab_size=8000, rate=0.1, config=None, logger=None): super(Transformer, self).__init__() if config is not None: num_enc_layers = config.model.N_encoder if logger is not None: logger.info('config.model.N_encoder: ' + str(num_enc_layers)) num_dec_layers = config.model.N_decoder if logger is not None: logger.info('config.model.N_decoder: ' + str(num_dec_layers)) d_model = config.model.d_model if logger is not None: logger.info('config.model.d_model: ' + str(d_model)) num_heads = config.model.n_heads if logger is not None: logger.info('config.model.n_heads: ' + str(num_heads)) dff = config.model.d_ff if logger is not None: logger.info('config.model.d_ff: ' + str(dff)) pe_max_len = config.model.pe_max_len if logger is not None: logger.info('config.model.pe_max_len:' + str(pe_max_len)) target_vocab_size = config.model.vocab_size if logger is not None: logger.info('config.model.vocab_size:' + str(target_vocab_size)) rate = config.model.dropout if logger is not None: logger.info('config.model.dropout: ' + str(rate)) else: print('use default params') num_enc_layers = num_layers num_dec_layers = num_layers self.encoder = Encoder(num_enc_layers, d_model, num_heads, dff, pe_max_len, 'encoder', rate) self.decoder = Decoder(num_dec_layers, d_model, num_heads, dff, target_vocab_size, 'decoder', pe_max_len, rate)
def __init__(self, x_dim, y_dim, mlp_hidden_size_list, latent_dim, use_rnn=False, use_self_attention=True, use_deter_path=True, self_attention_type="dot", cross_attention_type="multihead", **kwargs): super(NP_Model, self).__init__() self.x_dim = x_dim self.y_dim = y_dim self.mlp_hidden_size_list = mlp_hidden_size_list self.latent_dim = latent_dim self.use_rnn = use_rnn self.use_self_attention = use_self_attention self.use_deter_path = use_deter_path self.self_attention_type = self_attention_type self.cross_attention_type = cross_attention_type # NOTICE: Latent Encoder self._latent_encoder = LatentEncoder(input_x_dim=self.x_dim, input_y_dim=self.y_dim, hidden_dim_list=self.mlp_hidden_size_list, latent_dim=self.latent_dim) # NOTICE : Decoder self._decoder = Decoder(x_dim=self.x_dim, y_dim=self.y_dim, mid_hidden_dim_list=self.mlp_hidden_size_list, latent_dim=self.latent_dim, # the dim of last axis of sc and z.. use_deterministic_path=self.use_deter_path, # whether use d_path or not will change the size of input use_lstm=False) # NOTICE: Deterministic Encoder self._deter_encoder = DeterministicEncoder(input_x_dim=self.x_dim, input_y_dim=self.y_dim, hidden_dim_list=self.mlp_hidden_size_list, latent_dim=self.latent_dim, # the dim of last axis of r.. self_attention_type=self.self_attention_type, use_self_attn=self.use_self_attention, attention_layers=2, use_lstm=False, cross_attention_type='uniform', cross_attention_rep='identity', attention_dropout=0)
def __init__(self, cfg): super(MultiInstanceRecognition, self).__init__() self.cfg = cfg self.backbone = ResNet50(pretrained='torchvision://resnet50') self.neck = FPN(cfg['feature_channels'], cfg['fpn_out_channels'], LastLevelMaxPool()) self.roialign = MultiInstanceAlign(cfg['num_instances'], cfg['roi_size'], cfg['roi_feature_steps']) self.encoder = CnnEncoder( cfg['fpn_out_channels'] * cfg['num_instances'], cfg['encoder_channels']) self.voc_len = cfg['voc_len'] self.decoder = Decoder(self.cfg) self.num_instances = cfg['num_instances']
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=config.dropout_rate): super(Transformer, self).__init__() self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate) self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate) self.final_layer = tf.keras.layers.Dense(target_vocab_size)
def __init__(self,conv_in=1,conv_out=256,conv_kernel=9,pri_cap_num=8,pri_out=256,pri_kernel=3, pri_stride=2,digit_in=8,digit_out=16,num_classes=26,num_routes=256*2*2,share_weights=False, decoder_in=16*10,de_outputI=512,de_outputII=1024,de_outputIII=1200,device=torch.device('cuda')): super(CapsNet, self).__init__() decoder_in = decoder_in // 10 * num_classes # print(decoder_in) self.device = device # self.convLayer = ConvLayer(conv_in,conv_out,conv_kernel) self.alexNetLayer = AlexNet(num_classes) self.primary_capsLayer = PrimaryCaps(pri_cap_num,conv_out,pri_out,pri_kernel,pri_stride) self.digit_capsLayer = DigitCaps(digit_in,digit_out,num_classes,num_routes,share_weights) self.decoder = Decoder(decoder_in,de_outputI,de_outputII,de_outputIII,num_classes) self.dropout = nn.Dropout(p=0.3) self.batch_norm = nn.BatchNorm2d(num_features=3) self.relu = nn.ReLU() # for computing loss self.mse_loss = nn.MSELoss()
if __name__ == "__main__": # Directorio actual current_dir = os.path.realpath(os.path.dirname(__file__)) # Directorio donde se almacenarán los ficheros descargados y generados files_path = current_dir + '\\files' if not os.path.exists(files_path): os.mkdir(files_path) start = perf_counter() # Descarga del fichero xml_path, dtd_path = download_file(files_path) decoded_xml = files_path + '\\decoded-dblp.xml' # Decodificación del fichero decoder = Decoder(xml_path, decoded_xml, dtd_path) decoder.recode_file() # Obtención de los identificadores del fichero (España) scrapper = Scrapper() ids = scrapper.scrape(decoded_xml, mask='spain') # Descarga de datos de los autores crawler = Crawler() authors_data = {} pbar = tqdm(total=len(ids), desc="Descargando datos de los investigadores") for id in ids: authors, props = crawler.crawl(id) # Si al descargar los datos de un autor el servidor devuelve HTTP404 (Not found) se descarta ese autor if props is not None:
def create_model(vocab_src, vocab_tgt, config): encoder = Encoder(config) attention = create_attention(config) decoder = Decoder(attention, vocab_tgt.size(), config) model = CondNMT(vocab_src, vocab_tgt, encoder, decoder, config) return model
def __init__(self, input_channels, h_size, z_size): super(VAE, self).__init__() self.encoder = Encoder(input_channels, h_size, z_size) self.decoder = Decoder(input_channels, z_size)
class ModelGraph(object): def __init__(self, word_vocab=None, POS_vocab=None, NER_vocab=None, flags=None, mode='ce_train'): # mode can have the following values: # 'ce_train', # 'rl_train', # 'evaluate', # 'evaluate_bleu', # 'decode'. # it is different from mode in decoder which can be # 'ce_train', 'loss', 'greedy' or 'sample' self.mode = mode # is_training controls whether to use dropout is_training = True if mode in ('ce_train') else False self.flags = flags self.word_vocab = word_vocab # create placeholders self.create_placeholders() # create encoder self.encoder = Encoder(flags, self.passage_words, self.passage_POSs, self.passage_NERs, self.passage_lengths, self.answer_span, word_vocab=word_vocab, POS_vocab=POS_vocab, NER_vocab=NER_vocab) # encode the input instance self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.decoder_init_state = self.encoder.encode( is_training=is_training) max_passage_length = tf.shape(self.passage_words)[1] self.passage_mask = tf.sequence_mask(self.passage_lengths, max_passage_length, dtype=tf.float32) loss_weights = tf.sequence_mask( self.question_lengths, flags.max_question_len, dtype=tf.float32) # [batch_size, gen_steps] loss_weights_rl = tf.sequence_mask( self.question_lengths_rl, flags.max_question_len, dtype=tf.float32) # [batch_size, gen_steps] with tf.variable_scope("generator"): # create decoder self.decoder = Decoder(flags, word_vocab, self.rewards, is_training) if mode == 'decode': self.context_t_1 = tf.placeholder( tf.float32, [None, self.encoder_dim], name='context_t_1') # [batch_size, encoder_dim] self.coverage_t_1 = tf.placeholder( tf.float32, [None, None], name='coverage_t_1') # [batch_size, encoder_dim] self.word_t = tf.placeholder(tf.int32, [None], name='word_t') # [batch_size] (self.state_t, self.context_t, self.coverage_t, self.attn_dist_t, self.p_gen_t, self.ouput_t, self.topk_log_probs, self.topk_ids, self.greedy_prediction, self.multinomial_prediction) = self.decoder.decode( self.decoder_init_state, self.context_t_1, self.coverage_t_1, self.word_t, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask) # not buiding training op for this mode return elif mode == 'evaluate_bleu': _, _, self.greedy_words = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='greedy') # not buiding training op for this mode return elif mode in ('ce_train', 'evaluate'): self.accu, self.loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='ce_train') if mode == 'evaluate': # not buiding training op for evaluation return elif mode == 'rl_train': _, self.loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='loss') tf.get_variable_scope().reuse_variables() _, _, self.greedy_words = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, None, mode='greedy') elif mode == 'rl_ce_train': self.accu, self.ce_loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='ce_train') tf.get_variable_scope().reuse_variables() _, self.rl_loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs_rl, self.question_words_rl, loss_weights_rl, mode='loss') self.loss = BETA * self.ce_loss + self.rl_loss _, _, self.greedy_words = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, None, mode='greedy') # defining optimizer and train op optimizer = tf.train.AdagradOptimizer( learning_rate=flags.learning_rate) tvars = tf.trainable_variables() total_parameters = 0 for variable in tvars: shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters print("Total number of parameters is equal: %s" % total_parameters) if flags.lambda_l2 > 0.0: l2_loss = tf.add_n( [tf.nn.l2_loss(v) for v in tvars if v.get_shape().ndims > 1]) self.loss = self.loss + flags.lambda_l2 * l2_loss grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars), flags.clip_value) self.train_op = optimizer.apply_gradients(zip(grads, tvars)) ema = tf.train.ExponentialMovingAverage(decay=0.9999) with tf.control_dependencies([self.train_op]): self.train_op = ema.apply(tvars) with tf.variable_scope('backup_variables'): backup_vars = [ tf.get_variable(var.op.name, dtype=var.value().dtype, trainable=False, initializer=var.initialized_value()) for var in tvars ] save_backup_vars_op = tf.group( *(tf.assign(bck, var.read_value()) for var, bck in zip(tvars, backup_vars))) with tf.control_dependencies([save_backup_vars_op]): self.ema_to_vars_op = tf.group( *(tf.assign(var, ema.average(var).read_value()) for var in tvars)) self.restore_backup_vars_op = tf.group( *(tf.assign(var, bck.read_value()) for var, bck in zip(tvars, backup_vars))) def create_placeholders(self): # build placeholder for input passage/article self.passage_lengths = tf.placeholder(tf.int32, [None], name='passage_lengths') self.passage_words = tf.placeholder( tf.int32, [None, None], name="passage_words") # [batch_size, passage_len] self.passage_POSs = tf.placeholder( tf.int32, [None, None], name="passage_POSs") # [batch_size, passage_len] self.passage_NERs = tf.placeholder( tf.int32, [None, None], name="passage_NERs") # [batch_size, passage_len] # build placeholder for answer self.answer_span = tf.placeholder(tf.float32, [None, None], name="answer_span") # [batch_size] # build placeholder for question self.decoder_inputs = tf.placeholder( tf.int32, [None, self.flags.max_question_len], name="decoder_inputs") # [batch_size, gen_steps] self.question_words = tf.placeholder( tf.int32, [None, self.flags.max_question_len], name="question_words") # [batch_size, gen_steps] self.question_lengths = tf.placeholder( tf.int32, [None], name="question_lengths") # [batch_size] self.decoder_inputs_rl = tf.placeholder( tf.int32, [None, self.flags.max_question_len], name="decoder_inputs_rl") # [batch_size, gen_steps] self.question_words_rl = tf.placeholder( tf.int32, [None, self.flags.max_question_len], name="question_words_rl") # [batch_size, gen_steps] self.question_lengths_rl = tf.placeholder( tf.int32, [None], name="question_lengths_rl") # [batch_size] # build placeholder for reinforcement learning self.rewards = tf.placeholder(tf.float32, [None], name="rewards") def run_greedy(self, sess, batch): feed_dict = self.run_encoder(sess, batch, only_feed_dict=True) feed_dict[self.decoder_inputs] = batch.decoder_inputs return sess.run(self.greedy_words, feed_dict) def ce_train(self, sess, batch, only_eval=False): feed_dict = self.run_encoder(sess, batch, only_feed_dict=True) feed_dict[self.decoder_inputs] = batch.decoder_inputs feed_dict[self.question_words] = batch.question_words feed_dict[self.question_lengths] = batch.question_lengths if only_eval: return sess.run([self.accu, self.loss], feed_dict) else: return sess.run([self.train_op, self.loss], feed_dict)[1] def rl_train(self, sess, batch, with_ce): feed_dict = self.run_encoder(sess, batch, only_feed_dict=True) feed_dict[self.decoder_inputs] = batch.decoder_inputs greedy_outputs = sess.run(self.greedy_words, feed_dict) greedy_outputs = greedy_outputs.tolist() gold_output = batch.question_words.tolist() # baseline outputs by flipping coin flipp = 0.1 baseline_outputs = np.copy(batch.question_words) for i in range(batch.question_words.shape[0]): seq_len = min(self.flags.max_question_len, batch.question_lengths[i] - 1) # don't change stop token '</s>' for j in range(seq_len): if greedy_outputs[i][j] != 0 and random.random() < flipp: baseline_outputs[i, j] = greedy_outputs[i][j] baseline_outputs = baseline_outputs.tolist() rl_inputs = [] rl_outputs = [] rl_input_lengths = [] rewards = [] for i, (baseline_output, greedy_output) in enumerate( zip(baseline_outputs, greedy_outputs)): _, baseline_output_words = self.word_vocab.getLexical( baseline_output) greedy_output, greedy_output_words = self.word_vocab.getLexical( greedy_output) _, gold_output_words = self.word_vocab.getLexical(gold_output[i]) rl_inputs.append([int(batch.decoder_inputs[i, 0])] + greedy_output[:-1]) rl_outputs.append(greedy_output) rl_input_lengths.append(len(greedy_output)) baseline_output_words_list = baseline_output_words.split() greedy_output_words_list = greedy_output_words.split() gold_output_words_list = gold_output_words.split() if self.flags.reward_type == 'bleu': cc = SmoothingFunction() reward = sentence_bleu([gold_output_words_list], greedy_output_words_list, smoothing_function=cc.method3) baseline = sentence_bleu([gold_output_words_list], baseline_output_words_list, smoothing_function=cc.method3) rewards.append(reward - baseline) elif self.flags.reward_type == 'rouge': reward = rouge.rouge([gold_output_words], [greedy_output_words])["rouge_l/f_score"] baseline = rouge.rouge( [gold_output_words], [baseline_output_words])["rouge_l/f_score"] rewards.append(reward - baseline) else: raise ValueError("Reward type is not bleu or rouge!") rl_inputs = padding_utils.pad_2d_vals(rl_inputs, len(rl_inputs), self.flags.max_question_len) rl_outputs = padding_utils.pad_2d_vals(rl_outputs, len(rl_outputs), self.flags.max_question_len) rl_input_lengths = np.array(rl_input_lengths, dtype=np.int32) rewards = np.array(rewards, dtype=np.float32) #reward = rescale(reward) assert rl_inputs.shape == rl_outputs.shape feed_dict = self.run_encoder(sess, batch, only_feed_dict=True) feed_dict[self.rewards] = rewards if with_ce: feed_dict[self.decoder_inputs_rl] = rl_inputs feed_dict[self.question_words_rl] = rl_outputs feed_dict[self.question_lengths_rl] = rl_input_lengths feed_dict[self.decoder_inputs] = batch.decoder_inputs feed_dict[self.question_words] = batch.question_words feed_dict[self.question_lengths] = batch.question_lengths else: feed_dict[self.decoder_inputs] = rl_inputs feed_dict[self.question_words] = rl_outputs feed_dict[self.question_lengths] = rl_input_lengths _, loss = sess.run([self.train_op, self.loss], feed_dict) return loss def run_encoder(self, sess, batch, only_feed_dict=False): feed_dict = {} feed_dict[self.passage_lengths] = batch.sent1_length if self.flags.with_word: feed_dict[self.passage_words] = batch.sent1_word if self.flags.with_POS: feed_dict[self.passage_POSs] = batch.sent1_POS if self.flags.with_NER: feed_dict[self.passage_NERs] = batch.sent1_NER if self.flags.with_answer_span: feed_dict[self.answer_span] = batch.answer_span if only_feed_dict: return feed_dict return sess.run([ self.encoder_hidden_states, self.decoder_init_state, self.encoder_features, self.passage_words, self.passage_mask ], feed_dict)
def __init__(self, word_vocab=None, POS_vocab=None, NER_vocab=None, flags=None, mode='ce_train'): # mode can have the following values: # 'ce_train', # 'rl_train', # 'evaluate', # 'evaluate_bleu', # 'decode'. # it is different from mode in decoder which can be # 'ce_train', 'loss', 'greedy' or 'sample' self.mode = mode # is_training controls whether to use dropout is_training = True if mode in ('ce_train') else False self.flags = flags self.word_vocab = word_vocab # create placeholders self.create_placeholders() # create encoder self.encoder = Encoder(flags, self.passage_words, self.passage_POSs, self.passage_NERs, self.passage_lengths, self.answer_span, word_vocab=word_vocab, POS_vocab=POS_vocab, NER_vocab=NER_vocab) # encode the input instance self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.decoder_init_state = self.encoder.encode( is_training=is_training) max_passage_length = tf.shape(self.passage_words)[1] self.passage_mask = tf.sequence_mask(self.passage_lengths, max_passage_length, dtype=tf.float32) loss_weights = tf.sequence_mask( self.question_lengths, flags.max_question_len, dtype=tf.float32) # [batch_size, gen_steps] loss_weights_rl = tf.sequence_mask( self.question_lengths_rl, flags.max_question_len, dtype=tf.float32) # [batch_size, gen_steps] with tf.variable_scope("generator"): # create decoder self.decoder = Decoder(flags, word_vocab, self.rewards, is_training) if mode == 'decode': self.context_t_1 = tf.placeholder( tf.float32, [None, self.encoder_dim], name='context_t_1') # [batch_size, encoder_dim] self.coverage_t_1 = tf.placeholder( tf.float32, [None, None], name='coverage_t_1') # [batch_size, encoder_dim] self.word_t = tf.placeholder(tf.int32, [None], name='word_t') # [batch_size] (self.state_t, self.context_t, self.coverage_t, self.attn_dist_t, self.p_gen_t, self.ouput_t, self.topk_log_probs, self.topk_ids, self.greedy_prediction, self.multinomial_prediction) = self.decoder.decode( self.decoder_init_state, self.context_t_1, self.coverage_t_1, self.word_t, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask) # not buiding training op for this mode return elif mode == 'evaluate_bleu': _, _, self.greedy_words = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='greedy') # not buiding training op for this mode return elif mode in ('ce_train', 'evaluate'): self.accu, self.loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='ce_train') if mode == 'evaluate': # not buiding training op for evaluation return elif mode == 'rl_train': _, self.loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='loss') tf.get_variable_scope().reuse_variables() _, _, self.greedy_words = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, None, mode='greedy') elif mode == 'rl_ce_train': self.accu, self.ce_loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, loss_weights, mode='ce_train') tf.get_variable_scope().reuse_variables() _, self.rl_loss, _ = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs_rl, self.question_words_rl, loss_weights_rl, mode='loss') self.loss = BETA * self.ce_loss + self.rl_loss _, _, self.greedy_words = self.decoder.train( self.encoder_dim, self.encoder_hidden_states, self.encoder_features, self.passage_words, self.passage_mask, self.decoder_init_state, self.decoder_inputs, self.question_words, None, mode='greedy') # defining optimizer and train op optimizer = tf.train.AdagradOptimizer( learning_rate=flags.learning_rate) tvars = tf.trainable_variables() total_parameters = 0 for variable in tvars: shape = variable.get_shape() variable_parameters = 1 for dim in shape: variable_parameters *= dim.value total_parameters += variable_parameters print("Total number of parameters is equal: %s" % total_parameters) if flags.lambda_l2 > 0.0: l2_loss = tf.add_n( [tf.nn.l2_loss(v) for v in tvars if v.get_shape().ndims > 1]) self.loss = self.loss + flags.lambda_l2 * l2_loss grads, _ = tf.clip_by_global_norm(tf.gradients(self.loss, tvars), flags.clip_value) self.train_op = optimizer.apply_gradients(zip(grads, tvars)) ema = tf.train.ExponentialMovingAverage(decay=0.9999) with tf.control_dependencies([self.train_op]): self.train_op = ema.apply(tvars) with tf.variable_scope('backup_variables'): backup_vars = [ tf.get_variable(var.op.name, dtype=var.value().dtype, trainable=False, initializer=var.initialized_value()) for var in tvars ] save_backup_vars_op = tf.group( *(tf.assign(bck, var.read_value()) for var, bck in zip(tvars, backup_vars))) with tf.control_dependencies([save_backup_vars_op]): self.ema_to_vars_op = tf.group( *(tf.assign(var, ema.average(var).read_value()) for var in tvars)) self.restore_backup_vars_op = tf.group( *(tf.assign(var, bck.read_value()) for var, bck in zip(tvars, backup_vars)))
means = config["means"] std = config["stds"] channels = config["channel"] alpha = config["alpha"] csv_path = config["csv_path"] img_dir = config["image_dir"] output_dir = config["output_dir"] if not os.path.exists(output_dir): os.mkdir(output_dir) data = pd.read_csv(csv_path) paths = data["ImageId"].values paths = [os.path.join(img_dir, p) for p in paths] labels = data["TrueLabel"].values encoder = Encoder(channels, out_ch=2048) decoder = Decoder(2048, channels) encoder.load_state_dict(torch.load(config["encoder"], map_location="cpu")) decoder.load_state_dict(torch.load(config["decoder"], map_location="cpu")) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") encoder.to(device) decoder.to(device) encoder.eval() decoder.eval() x_adv = [] with torch.no_grad(): bar = tqdm.tqdm(paths) for path in bar: filename = os.path.basename(path)
def __init__(self): super().__init__() self.encoder = Encoder() self.decoder = Decoder() self.egm = EdgeGuidanceModule() self.wam = WeightedAggregationModule()
# tsf.ToTensor(), # tsf.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)) # ]) # train_dataset = CIFAR10("./data", train=True, transform=train_transform, download=True) # test_dataset = CIFAR10("./data", train=False, transform=val_transform, download=False) #train_paths, val_paths, train_labels, val_labels = train_test_split(paths, labels, random_state=0, stratify=labels) train_paths, train_labels = get_paths("./cat_dog/train", img_suffix=".jpg") val_paths, val_labels = get_paths("./cat_dog/val", img_suffix='.jpg') train_dataset = PALMClassifyDataset(train_paths, train_labels, augmentation=True, img_size=img_size) test_dataset = PALMClassifyDataset(val_paths, val_labels, augmentation=False, img_size=img_size) train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=num_workers) test_loader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers) encoder = Encoder(in_ch=channels, out_ch=2048) decoder = Decoder(in_ch=2048, out_ch=channels) classifier = ResNet(channels, n_layers=50, num_classes=num_classes) #classifier = resnet18(pretrained=False, num_classes=num_classes, zero_init_residual=False) # classifier.load_state_dict(torch.load("./best_classifier.pth", map_location="cpu")) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") encoder.to(device) decoder.to(device) classifier.to(device) encoder_opt = opt.Adam(encoder.parameters(), lr=init_lr, weight_decay=5e-4) decoder_opt = opt.Adam(decoder.parameters(), lr=init_lr, weight_decay=5e-4) classifier_opt = opt.Adam(classifier.parameters(), lr=init_lr, weight_decay=5e-4) c_loss = nn.CrossEntropyLoss()