def __init__(self, config, input_emb_size, mix_speech_len, tgt_vocab_size, use_cuda, pretrain=None, score_fn=''): super(seq2seq, self).__init__() if pretrain is not None: src_embedding = pretrain['src_emb'] tgt_embedding = pretrain['tgt_emb'] else: src_embedding = None tgt_embedding = None self.encoder = models.rnn_encoder(config, input_emb_size, None, embedding=src_embedding) if config.shared_vocab == False: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=tgt_embedding, score_fn=score_fn) else: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=self.encoder.embedding, score_fn=score_fn) self.use_cuda = use_cuda self.tgt_vocab_size = tgt_vocab_size self.config = config self.criterion = models.criterion(tgt_vocab_size, use_cuda,config.loss) self.loss_for_ss = nn.MSELoss() self.log_softmax = nn.LogSoftmax() self.wav_loss = models.WaveLoss(dBscale=1, nfft=config.FRAME_LENGTH, hop_size=config.FRAME_SHIFT) speech_fre = input_emb_size num_labels = tgt_vocab_size if config.use_tas: self.ss_model = models.ConvTasNet() else: self.ss_model = models.SS(config, speech_fre, mix_speech_len, num_labels)
def __init__(self, config, use_attention=True, encoder=None, decoder=None): super(seq2seq, self).__init__() if encoder is not None: self.encoder = encoder else: if config.sae: self.encoder = models.gated_rnn_encoder(config) self.encoder_ae = models.rnn_decoder(config) else: self.encoder = models.rnn_encoder(config) self.score_fn = config.score_fn tgt_embedding = self.encoder.embedding if config.shared_vocab else None if decoder is not None: self.decoder = decoder else: self.decoder = models.rnn_decoder(config, embedding=tgt_embedding, use_attention=use_attention, score_fn=self.score_fn) self.log_softmax = nn.LogSoftmax(dim=-1) self.use_cuda = config.use_cuda self.config = config self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduce=False) if config.use_cuda: self.criterion.cuda()
def __init__(self, config, src_vocab_size, tgt_vocab_size, use_cuda, pretrain=None, score_fn=None): super(seq2seq, self).__init__() if pretrain is not None: src_embedding = pretrain['src_emb'] tgt_embedding = pretrain['tgt_emb'] else: src_embedding = None tgt_embedding = None self.encoder = models.rnn_encoder(config, src_vocab_size, embedding=src_embedding) if config.shared_vocab == False: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=tgt_embedding, score_fn=score_fn) else: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=self.encoder.embedding, score_fn=score_fn) self.use_cuda = use_cuda self.src_vocab_size = src_vocab_size self.tgt_vocab_size = tgt_vocab_size self.config = config self.criterion = models.criterion(tgt_vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax()
def __init__(self, config, src_vocab_size, tgt_vocab_size, use_cuda, w2v=None, score_fn=None, weight=0.0, pretrain_updates=0, extend_vocab_size=0, device_ids=None): super(ae, self).__init__() if w2v is not None: src_embedding = w2v['src_emb'] tgt_embedding = w2v['tgt_emb'] else: src_embedding = None tgt_embedding = None self.encoder_s2s = models.gated_rnn_encoder(config, src_vocab_size, embedding=src_embedding) if config.shared_vocab == False: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=tgt_embedding, score_fn=score_fn) else: self.decoder = models.rnn_decoder( config, tgt_vocab_size, embedding=self.encoder_s2s.embedding, score_fn=score_fn) self.encoder_ae = models.rnn_encoder(config, src_vocab_size, embedding=self.decoder.embedding) self.use_cuda = use_cuda self.src_vocab_size = src_vocab_size self.tgt_vocab_size = tgt_vocab_size self.config = config self.weight = weight self.pretrain_updates = pretrain_updates if 'emb' in score_fn: self.criterion = models.criterion_emb(config.hidden_size, tgt_vocab_size, use_cuda) else: self.criterion = models.criterion(tgt_vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax() if score_fn.startswith('dis'): self.discriminator = nn.Linear( config.num_layers * config.hidden_size * 2, 1) self.sigmoid = nn.Sigmoid() if score_fn.endswith('map'): self.h_map = nn.Linear(config.hidden_size, config.hidden_size) self.c_map = nn.Linear(config.hidden_size, config.hidden_size) self.score_fn = score_fn
def __init__(self, config, src_vocab_size, tgt_vocab_size, use_cuda, pretrain=None, score_fn=None): super(seq2seq, self).__init__() if pretrain is not None: # hint: 会自动冻结 src_embedding = nn.Embedding.from_pretrained(pretrain['src_emb']) tgt_embedding = nn.Embedding.from_pretrained(pretrain['tgt_emb']) ''' def normal2(A): return A / np.sqrt(np.sum(A ** 2)) # for i in range(len(pretrain['tgt_emb'])): # pretrain['tgt_emb'][i] = normal2(pretrain['tgt_emb'][i]) mat = np.zeros(45*45).reshape(45, 45) for i in range(45): for j in range(45): _ = normal2(pretrain['tgt_emb'][i].numpy().copy()) __ = normal2(pretrain['tgt_emb'][j].numpy().copy()) mat[i][j] = _.dot(__) print(mat) print() ''' else: src_embedding = None tgt_embedding = None self.encoder = models.rnn_encoder(config, src_vocab_size, embedding=src_embedding, tgt_embedding=tgt_embedding) if config.shared_vocab == False: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=tgt_embedding, score_fn=score_fn) else: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=self.encoder.embedding, score_fn=score_fn) self.use_cuda = use_cuda self.src_vocab_size = src_vocab_size self.tgt_vocab_size = tgt_vocab_size self.config = config if config.score == 'margin': # print("using margin loss") self.criterion = models.margin_criterion(tgt_vocab_size, use_cuda, config) elif config.score == 'hybrid': self.criterion = {} self.criterion['softmax'] = models.criterion(tgt_vocab_size, use_cuda, config) self.criterion['margin'] = models.margin_criterion(tgt_vocab_size, use_cuda, config) elif config.score == 'softmax': self.criterion = models.criterion(tgt_vocab_size, use_cuda, config) else: print('no such score function') os.abort() self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, config, use_attention=True, encoder=None, decoder=None, src_padding_idx=0, tgt_padding_idx=0): super(seq2seq, self).__init__() if encoder is not None: self.encoder = encoder else: self.encoder1 = models.rnn_encoder(config, padding_idx=src_padding_idx) self.encoder2 = models.rnn_encoder(config, padding_idx=src_padding_idx) tgt_embedding = self.encoder1.embedding if config.shared_vocab else None if decoder is not None: self.decoder = decoder else: self.decoder = models.rnn_decoder(config, embedding=tgt_embedding, \ use_attention=use_attention, padding_idx=tgt_padding_idx) self.log_softmax = nn.LogSoftmax(dim=-1) self.use_cuda = config.use_cuda self.config = config self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduction='none') if config.use_cuda: self.criterion.cuda()
def __init__(self, config, use_attention=True, encoder=None, decoder=None): super(seq2seq, self).__init__() if encoder is not None: self.encoder = encoder else: if config.transformer: # n_src_vocab, len_max_seq, d_word_vec,n_layers, n_head, d_k, d_v,d_model, d_inner, self.encoder = Encoder(config.src_vocab_size, 100, config.emb_size, 6, 8, int(config.hidden_size / 8), int(config.hidden_size / 8), config.hidden_size, config.hidden_size * 4) else: self.encoder = models.rnn_encoder(config) tgt_embedding = self.encoder.embedding if config.shared_vocab else None if decoder is not None: self.decoder = decoder else: self.decoder = models.rnn_decoder(config, embedding=tgt_embedding, use_attention=use_attention) self.log_softmax = nn.LogSoftmax(dim=-1) self.use_cuda = config.use_cuda self.config = config self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduction='none') if config.use_cuda: self.criterion.cuda()
def __init__(self, config, vocab, use_cuda, pretrain=None): super(hierarchical_attention, self).__init__() self.vocab = vocab self.vocab_size = vocab.voc_size if pretrain is not None: self.embedding = pretrain['emb'] else: self.embedding = nn.Embedding(self.vocab_size, config.emb_size) self.word_encoder = Encoder(config.emb_size, config.emb_size, config.num_layers, config.dropout, config.bidirec) self.word_attentive_pool = attentive_pooling(config.emb_size * 2) self.sentence_encoder = Encoder(config.emb_size * 2, config.emb_size * 2, config.num_layers, config.dropout, config.bidirec) self.sentence_attentive_pool = attentive_pooling( config.decoder_hidden_size) self.decoder = models.rnn_decoder(config, self.vocab_size, embedding=self.embedding) self.w_context = nn.Linear(config.decoder_hidden_size * 2, config.decoder_hidden_size, bias=False) self.state_wc = nn.Linear( config.decoder_hidden_size, config.decoder_hidden_size * config.num_layers) self.state_wh = nn.Linear( config.decoder_hidden_size, config.decoder_hidden_size * config.num_layers) self.tanh = nn.Tanh() self.config = config self.criterion = models.criterion(self.vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax()
def __init__(self, opt): super(seq2seq, self).__init__() self.opt = opt self.encoder = models.rnn_encoder(opt) self.decoder = models.rnn_decoder(opt) self.criterion = models.criterion(opt.use_cuda) self.log_softmax = nn.LogSoftmax(dim=1)
def __init__(self, config, src_vocab, tgt_vocab, use_cuda, bmodel, pretrain=None, score_fn=None): super(seq2seq, self).__init__() if pretrain is not None: self.slot_embedding = nn.Embedding.from_pretrained( pretrain['slot'], freeze=False) else: self.slot_embedding = None if bert: self.src_embedding = bmodel self.dsep_id = src_vocab['-'] self.ssep_id = src_vocab[','] self.vsep_id = src_vocab[';'] src_vocab_size = len(src_vocab) tgt_vocab_size = len(tgt_vocab) self.encoder = models.rnn_encoder( config, ) # tgt_vocab_size,False, embedding=slot_embedding) self.decoder = models.rnn_decoder(config, src_vocab_size, tgt_vocab_size, slot_embedding=self.slot_embedding, value_embedding=self.slot_embedding, score_fn=score_fn) self.use_cuda = use_cuda self.config = config self.criterion = models.criterion(tgt_vocab_size, use_cuda)
def __init__(self, config, vocab, use_cuda, pretrain=None): super(bow2seq, self).__init__() self.vocab = vocab self.vocab_size = vocab.voc_size if pretrain is not None: self.embedding = pretrain['emb'] else: self.embedding = nn.Embedding(self.vocab_size, config.emb_size) # self.encoder = models.GCN_Encoder(config, self.vocab_size, embedding=self.embedding) self.bert_encoder = models.bert.BERT(config.head_num, config.emb_size, config.dropout, config.emb_size, self.vocab_size, config.num_layers, config.max_sentence_len, word_emb=self.embedding) self.decoder = models.rnn_decoder(config, self.vocab_size, embedding=self.embedding) self.proj = nn.Linear(config.emb_size, config.decoder_hidden_size) self.state_wc = nn.Linear( config.decoder_hidden_size, config.decoder_hidden_size * config.num_layers) self.state_wh = nn.Linear( config.decoder_hidden_size, config.decoder_hidden_size * config.num_layers) self.tanh = nn.Tanh() self.config = config self.criterion = models.criterion(self.vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax()
def __init__(self, config, use_attention=True, encoder=None, decoder=None, src_padding_idx=0, tgt_padding_idx=0, label_smoothing=0, tgt_vocab=None): super(seq2seq, self).__init__() if encoder is not None: self.encoder = encoder else: self.encoder = models.rnn_encoder( config, padding_idx=src_padding_idx) tgt_embedding = self.encoder.embedding if config.shared_vocab else None if decoder is not None: self.decoder = decoder else: self.decoder = models.rnn_decoder( config, embedding=tgt_embedding, use_attention=use_attention, padding_idx=tgt_padding_idx) self.log_softmax = nn.LogSoftmax(dim=-1) self.use_cuda = config.use_cuda self.config = config self.label_smoothing = label_smoothing if self.label_smoothing > 0: self.criterion = LabelSmoothingLoss( label_smoothing, config.tgt_vocab_size, ignore_index=tgt_padding_idx) else: self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduction='none') if config.use_cuda: self.criterion.cuda() if config.rl: # self.reward_provider = CTRRewardProvider( # config.ctr_rewared_provider_path) self.tgt_vocab = tgt_vocab self.padding_idx = tgt_padding_idx
def __init__(self, config, vocab, use_cuda, use_content=False, pretrain=None): super(seq2seq, self).__init__() self.vocab = vocab self.vocab_size = vocab.voc_size if pretrain is not None: self.embedding = pretrain['emb'] else: self.embedding = nn.Embedding(self.vocab_size, config.emb_size) if config.emb_file is not None: print(len(vocab._word2id)) print('Loading embedding file: %s' % config.emb_file) embeddings = np.random.randn(self.vocab_size, config.emb_size) * 0.01 pre_trained = 0 i = 0 #small = open('data/music/small_embedding.txt', 'w') count = 0 #with open('data/music/small_embedding.txt', 'w') as small: for line in open(config.emb_file, 'r', encoding='utf-8').readlines(): count += 1 if count % 100000 == 0: print(count) sp = line.split() if (len(sp) == config.emb_size + 1) and sp[0] in set( vocab._id2word): pre_trained += 1 embeddings[vocab._word2id[sp[0]]] = [ float(x) for x in sp[1:] ] #small.write(line) else: i += 1 #print(sp[0]) #small.close() print("Number of len(sp)!=301 :", i) print('Pre-trained: %d (%.2f%%)' % (pre_trained, pre_trained * 100.0 / len(vocab._word2id))) self.embedding.weight.data.copy_(torch.FloatTensor(embeddings)) self.encoder = models.rnn_encoder(config, self.vocab_size, embedding=self.embedding) self.decoder = models.rnn_decoder(config, self.vocab_size, embedding=self.embedding) self.config = config self.use_content = use_content self.criterion = models.criterion(self.vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax() self.tanh = nn.Tanh()
def __init__(self, config, input_emb_size, mix_speech_len, tgt_vocab_size, use_cuda, pretrain=None, score_fn=None): super(seq2seq, self).__init__() if pretrain is not None: src_embedding = pretrain['src_emb'] tgt_embedding = pretrain['tgt_emb'] else: src_embedding = None tgt_embedding = None self.encoder = models.rnn_encoder(config, input_emb_size, None, embedding=src_embedding) if config.shared_vocab == False: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=tgt_embedding, score_fn=score_fn) else: self.decoder = models.rnn_decoder(config, tgt_vocab_size, embedding=self.encoder.embedding, score_fn=score_fn) self.use_cuda = use_cuda self.tgt_vocab_size = tgt_vocab_size self.config = config self.criterion = models.criterion(tgt_vocab_size, use_cuda) self.loss_for_ss = nn.MSELoss() self.log_softmax = nn.LogSoftmax() speech_fre = input_emb_size num_labels = tgt_vocab_size self.ss_model = models.SS(config, speech_fre, mix_speech_len, num_labels)
def __init__(self, config, vocab, use_cuda, use_copy, use_bert, word_level_model, graph_model, pretrain=None): super(graph2seq, self).__init__() self.word_level_model = word_level_model self.vocab = vocab self.vocab_size = vocab.voc_size if pretrain is not None: self.embedding = pretrain['emb'] else: self.embedding = nn.Embedding(self.vocab_size, config.emb_size) # self.encoder = models.GCN_Encoder(config, self.vocab_size, embedding=self.embedding) self.use_copy = use_copy self.use_bert = use_bert if use_bert: self.bert_encoder = models.bert.BERT( config.head_num, config.decoder_hidden_size, config.dropout, config.decoder_hidden_size, self.vocab_size, config.num_layers, config.max_sentence_len) self.encoder = models.Memory_Network(config, self.vocab_size, word_level_model, graph_model, embedding=self.embedding) if use_copy: self.decoder = models.pointer_decoder(config, self.vocab_size, embedding=self.embedding) else: self.decoder = models.rnn_decoder(config, self.vocab_size, embedding=self.embedding, gru=False) self.state_wc = nn.Linear( config.decoder_hidden_size, config.decoder_hidden_size * config.num_layers) self.state_wh = nn.Linear( config.decoder_hidden_size, config.decoder_hidden_size * config.num_layers) self.tanh = nn.Tanh() self.config = config self.criterion = models.criterion(self.vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax()
def __init__(self, config, use_attention=True, encoder=None, decoder=None): super(seq2seq, self).__init__() # 如果encoder为None,则创建rnn_encoder if encoder is not None: self.encoder = encoder else: self.encoder = models.rnn_encoder(config) # 如果是共享词表,则tgt_emb等于encoder的embedding tgt_embedding = self.encoder.embedding if config.shared_vocab else None if decoder is not None: self.decoder = decoder else: self.decoder = models.rnn_decoder(config, embedding=tgt_embedding, use_attention=use_attention) self.log_softmax = nn.LogSoftmax(dim=-1) self.use_cuda = config.use_cuda self.config = config self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduction='none') if config.use_cuda: self.criterion.cuda()
def __init__(self, config, vocab, use_cuda, use_content=False, pretrain=None): super(seq2seq, self).__init__() self.vocab = vocab self.vocab_size = vocab.voc_size if pretrain is not None: self.embedding = pretrain['emb'] else: self.embedding = nn.Embedding(self.vocab_size, config.emb_size) self.encoder = models.rnn_encoder(config, self.vocab_size, embedding=self.embedding) self.decoder = models.rnn_decoder(config, self.vocab_size, embedding=self.embedding) self.config = config self.use_content = use_content self.criterion = models.criterion(self.vocab_size, use_cuda) self.log_softmax = nn.LogSoftmax() self.tanh = nn.Tanh()
def __init__(self, config, use_attention=True, encoder=None, decoder=None): super(seq2seq, self).__init__() if encoder is not None: self.encoder = encoder else: self.encoder = models.rnn_encoder(config) tgt_embedding = self.encoder.embedding if config.shared_vocab else None if decoder is not None: self.decoder = decoder else: self.decoder = models.rnn_decoder(config, embedding=tgt_embedding, use_attention=use_attention) self.log_softmax = nn.LogSoftmax(dim=-1) self.softmax = nn.Softmax(dim=-1) self.use_cuda = config.use_cuda self.config = config weight = torch.ones(config.tgt_vocab_size) weight[0] = 0 self.criterion = nn.CrossEntropyLoss(ignore_index=utils.PAD, reduce=False) self.multi_label_loss = nn.MultiLabelSoftMarginLoss(weight=weight, size_average=False)