def __init__(self, enc_inp_size, d_latent, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1, device='cpu', d_map_latent=8): super(EncoderY, self).__init__() self.d_model = d_model self.embed_fn = nn.Sequential( LinearEmbedding(enc_inp_size, d_model - d_map_latent), PositionalEncoding(d_model - d_map_latent, dropout)) self.encoder = Encoder( EncoderLayer(d_model, MultiHeadAttention(h, d_model), PointerwiseFeedforward(d_model, d_ff, dropout), dropout), N) self.fc = nn.Linear(d_model, d_latent) self.init_weights(self.encoder.parameters()) self.init_weights(self.fc.parameters()) self.map_encoder = load_map_encoder(device)
def create_model(cls, args): from transformer.conv_encoder import Conv2dSubsample from transformer.encoder import Encoder from transformer.attentionAssigner import Attention_Assigner # from transformer.attentionAssigner import Attention_Assigner_RNN as Attention_Assigner from transformer.decoder import Decoder_CIF as Decoder conv_encoder = Conv2dSubsample(d_input=args.d_input * args.LFR_m, d_model=args.d_model, n_layers=args.n_conv_layers) encoder = Encoder(d_input=args.d_model, n_layers=args.n_layers_enc, n_head=args.n_head, d_model=args.d_model, d_inner=args.d_inner, dropout=args.dropout) assigner = Attention_Assigner(d_input=args.d_model, d_hidden=args.d_assigner_hidden, w_context=args.w_context, n_layers=args.n_assigner_layers) decoder = Decoder(sos_id=args.sos_id, n_tgt_vocab=args.vocab_size, n_layers=args.n_layers_dec, n_head=args.n_head, d_model=args.d_model, d_inner=args.d_inner, dropout=args.dropout) model = cls(conv_encoder, encoder, assigner, decoder, args.spec_aug_cfg) return model
def __init__(self, src_len, tgt_len, enc_inp_size, dec_inp_size, dec_out_size, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1, device='cpu'): super(Generator, self).__init__() self.device = device self.src_len = src_len self.tgt_len = tgt_len self.dec_inp_size = dec_inp_size c = copy.deepcopy attn = MultiHeadAttention(h, d_model) ff = PointerwiseFeedforward(d_model, d_ff, dropout) position = PositionalEncoding(d_model, dropout) self.generator = EncoderDecoder( Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N), Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N), nn.Sequential(LinearEmbedding(enc_inp_size, d_model), c(position)), nn.Sequential(LinearEmbedding(dec_inp_size, d_model), c(position)), TFHeadGenerator(d_model, dec_out_size)) # This was important from their code. # Initialize parameters with Glorot / fan_avg. for p in self.generator.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p)
def make_model( src_vocab: int, tgt_vocab: int, n: int = 6, d_model: int = 512, d_ff: int = 2048, h: int = 8, dropout: float = 0.1, device: torch.device = torch.device("cpu"), ) -> EncoderDecoder: """Helper: Construct a model from hyperparameters.""" c = copy.deepcopy attn = MultiHeadedAttention(h, d_model) ff = PositionwiseFeedForward(d_model, d_ff, dropout) position = PositionalEncoding(d_model, dropout) model = EncoderDecoder( Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), n), Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), n), nn.Sequential(Embeddings(d_model, src_vocab), c(position)), nn.Sequential(Embeddings(d_model, tgt_vocab), c(position)), Generator(d_model, tgt_vocab), ).to(device) # This was important from their code. # Initialize parameters with Glorot / fan_avg. for p in model.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p) return model
def __init__(self, enc_inp_size, dec_inp_size, dec_out_size, N=6, d_model=512, d_ff=2048, heads=8, dropout=0.1, mean=[0, 0], std=[0, 0]): super(IndividualTF, self).__init__() "Helper: Construct a model from hyperparameters." c = copy.deepcopy attn = MultiHeadAttention(heads, d_model) ff = PointerwiseFeedforward(d_model, d_ff, dropout) position = PositionalEncoding(d_model, dropout) self.mean = np.array(mean) self.std = np.array(std) self.model = EncoderDecoder( Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N), Decoder(DecoderLayer(d_model, c(attn), c(attn), c(ff), dropout), N), nn.Sequential(LinearEmbedding(enc_inp_size, d_model), c(position)), nn.Sequential(LinearEmbedding(dec_inp_size, d_model), c(position)), Generator(d_model, dec_out_size)) # This was important from their code. # Initialize parameters with Glorot / fan_avg. for p in self.model.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p)
def create_model(cls, args): from transformer.decoder import Decoder from transformer.encoder import Encoder from transformer.conv_encoder import Conv2dSubsample conv_encoder = Conv2dSubsample(d_input=args.d_input * args.LFR_m, d_model=args.d_model, n_layers=args.n_conv_layers) encoder = Encoder(d_input=args.d_model, n_layers=args.n_layers_enc, n_head=args.n_head, d_model=args.d_model, d_inner=args.d_inner, dropout=args.dropout) decoder = Decoder(sos_id=args.sos_id, eos_id=args.eos_id, n_tgt_vocab=args.vocab_size, n_layers=args.n_layers_dec, n_head=args.n_head, d_model=args.d_model, d_inner=args.d_inner, dropout=args.dropout) model = cls(conv_encoder, encoder, decoder, spec_aug_cfg=args.spec_aug_cfg) return model
def __init__(self, disc_inp_size, disc_seq_len, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1, device='cpu'): super(Critic, self).__init__() self.device = device c = copy.deepcopy attn = MultiHeadAttention(h, d_model) ff = PointerwiseFeedforward(d_model, d_ff, dropout) position = PositionalEncoding(d_model, dropout) self.critic = nn.ModuleDict({ 'src_embed': nn.Sequential(LinearEmbedding(disc_inp_size, d_model), c(position)), 'encoder': Encoder(EncoderLayer(d_model, c(attn), c(ff), dropout), N), 'disc_head': nn.Sequential(nn.Flatten(), nn.Linear(d_model * disc_seq_len, 1)), }) for p in self.critic.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p)
def __init__(self, vocabulary_size_in, vocabulary_size_out, constants, hyperparams): super(Transformer, self).__init__() self.constants = constants self.max_seq = hyperparams.MAX_SEQ self.EmbeddingSrc = Embedding(vocabulary_size=vocabulary_size_in, d_model=hyperparams.D_MODEL, constants=constants) self.EmbeddingTgt = Embedding(vocabulary_size=vocabulary_size_out, d_model=hyperparams.D_MODEL, constants=constants) self.Encoder = Encoder(nb_layers=hyperparams.NB_LAYERS, nb_heads=hyperparams.NB_HEADS, d_model=hyperparams.D_MODEL, nb_neurons=hyperparams.NB_NEURONS, dropout=hyperparams.DROPOUT) self.Decoder = Decoder(nb_layers=hyperparams.NB_LAYERS, nb_heads=hyperparams.NB_HEADS, d_model=hyperparams.D_MODEL, nb_neurons=hyperparams.NB_NEURONS, dropout=hyperparams.DROPOUT) self.Linear = nn.Linear(hyperparams.D_MODEL, vocabulary_size_out, bias=False) if hyperparams.SHARE_WEIGHTS: self.EmbeddingSrc.lookup_table.weight = self.Linear.weight self.EmbeddingTgt.lookup_table.weight = self.Linear.weight
def test_forward(self): enc_layer = EncoderLayer( size=512, self_attention=MultiHeadAttention(n_head=8, d_model=512, d_k=64, d_v=64, dropout=0.1), feed_forward=PositionwiseFeedForward(d_model=512, d_ff=2048, dropout=0.1), dropout=0.1) encoder = Encoder(layer=enc_layer, n_layers=6) x = torch.ones((64, 10, 512)) out = encoder.forward(x, mask=None, verbose=False) self.assertIsInstance(out, torch.Tensor) self.assertEqual(out.shape, x.shape) # check no nan values self.assertEqual(torch.isnan(out).sum(), 0)
class EncoderX(nn.Module): def __init__(self, enc_inp_size, d_latent, N=6, d_model=512, d_ff=2048, h=8, dropout=0.1, device='cpu', d_map_latent=8): super(EncoderX, self).__init__() self.d_model = d_model self.embed_fn = nn.Sequential( LinearEmbedding(enc_inp_size, d_model - d_map_latent), PositionalEncoding(d_model - d_map_latent, dropout)) self.encoder = Encoder( EncoderLayer(d_model, MultiHeadAttention(h, d_model), PointerwiseFeedforward(d_model, d_ff, dropout), dropout), N) self.fc = nn.Linear(d_model, d_latent) self.init_weights(self.encoder.parameters()) self.init_weights(self.fc.parameters()) self.map_encoder = load_map_encoder(device) def init_weights(self, params): for p in params: if p.dim() > 1: nn.init.xavier_uniform_(p) def forward(self, src, src_mask, map): # map = map.view(-1, map.shape[2], map.shape[3], map.shape[4]) map_feat = self.map_encoder(src.reshape(-1, src.shape[2]), map.reshape(-1, map.shape[2], map.shape[3], map.shape[4]), train=False) map_feat = map_feat.reshape((-1, 8, map_feat.shape[-1])) logit_token = Variable( torch.FloatTensor(np.random.rand(src.shape[0], 1, self.d_model))).to(src.device) src_emb = torch.cat((self.embed_fn(src), map_feat), dim=-1) logit_src_emb = torch.cat((logit_token, src_emb), dim=1) enc_out = self.encoder(logit_src_emb, src_mask) # bs, 1+8, 512 logit = self.fc(enc_out[:, 0]) return enc_out[:, 1:], logit
def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=0.1): super(Transformer, self).__init__() self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate) self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate) self.final_layer = tf.keras.layers.Dense(target_vocab_size)
def __init__(self, n_src_vocab, n_trg_vocab, src_pad_idx, trg_pad_idx, d_word_vec=256, d_model=256, d_inner=512, n_layer=3, n_head=8, dropout=0.1, n_position=200): super(Transformer, self).__init__() self.src_pad_idx = src_pad_idx self.trg_pad_idx = trg_pad_idx self.encoder = Encoder(n_src_vocab=n_src_vocab, d_word_vec=d_word_vec, d_model=d_model, d_inner=d_inner, n_layer=n_layer, n_head=n_head, pad_idx=src_pad_idx, dropout=dropout, n_position=n_position, max_seq_len=32) self.decoder = Decoder(n_trg_vocab=n_trg_vocab, d_word_vec=d_word_vec, d_model=d_model, d_inner=d_inner, n_layer=n_layer, n_head=n_head, pad_idx=trg_pad_idx, n_position=n_position, dropout=dropout) self.trg_word_prj = nn.Linear(d_model, n_trg_vocab, bias=False) # for name, param in self.named_parameters(): # if param.dim() > 1: # nn.init.xavier_normal(param) for param in self.parameters(): if param.dim() > 1: nn.init.xavier_uniform_(param)
def __init__(self, *args, embedding_rank=None, inner_rank=None, ffward_rank=None, **kwargs): # Run super constructor from NMTModel, but don't run NMTModel.__init__ super(NMTModel, self).__init__() self.vocab = pickle.load(open(paths.vocab, 'rb')) if embedding_rank is None: embedding_rank = transformer_config.embedding_rank if inner_rank is None: inner_rank = transformer_config.inner_rank if ffward_rank is None: ffward_rank = transformer_config.ffward_rank print(transformer_config.embedding_factorization, transformer_config.inner_factorization, transformer_config.ffward_factorization) print(embedding_rank, inner_rank, ffward_rank) self.encoder = Encoder(len(self.vocab.src), embedding_rank, inner_rank, ffward_rank) self.decoder = Decoder(len(self.vocab.tgt), embedding_rank, inner_rank, ffward_rank) self.gpu = False self.initialize() self.optimizer = NoamOpt(transformer_config.layer_dimension, train_config.lr, 4000, Adam( self.parameters(), lr=0, betas=(0.9, 0.98), eps=1e-9, ), beginning_step=0) self.num_accumulations = 0 self.accumulate = max(1, train_config.accumulate)
def create_model(cls, args): from transformer.decoder import Decoder from transformer.encoder import Encoder encoder = Encoder(d_input=args.d_input * args.LFR_m, n_layers=args.n_layers_enc, n_head=args.n_head, d_model=args.d_model, d_inner=args.d_inner, dropout=args.dropout) decoder = Decoder(sos_id=args.sos_id, eos_id=args.eos_id, n_tgt_vocab=args.vocab_size, n_layers=args.n_layers_dec, n_head=args.n_head, d_model=args.d_model, d_inner=args.d_inner, dropout=args.dropout) model = cls.create_model(encoder, decoder) return model
def __init__(self, src_vocab_size, tgt_vocab_size, device, d_model=512, p_dropout=0.1): super(Transformer, self).__init__() self.d_model = d_model self.src_embedding = nn.Embedding(src_vocab_size, d_model) self.positional_encoder1 = PositionalEncoder(device, d_model=d_model, p_dropout=p_dropout) self.tgt_embedding = nn.Embedding(tgt_vocab_size, d_model) self.positional_encoder2 = PositionalEncoder(device, d_model=d_model, p_dropout=p_dropout) self.encoder = Encoder(6, d_model) self.decoder = Decoder(6, d_model) self.linear = nn.Linear(d_model, tgt_vocab_size) self.softmax = nn.LogSoftmax(dim=-1) # Share weights self.linear.weight = self.tgt_embedding.weight
def train_net(args): torch.manual_seed(7) np.random.seed(7) checkpoint = args.checkpoint start_epoch = 0 best_loss = float('inf') writer = SummaryWriter() epochs_since_improvement = 0 # Initialize / load checkpoint if checkpoint is None: # model encoder = Encoder(n_src_vocab, args.n_layers_enc, args.n_head, args.d_k, args.d_v, args.d_model, args.d_inner, dropout=args.dropout, pe_maxlen=args.pe_maxlen) decoder = Decoder( sos_id, eos_id, n_tgt_vocab, args.d_word_vec, args.n_layers_dec, args.n_head, args.d_k, args.d_v, args.d_model, args.d_inner, dropout=args.dropout, tgt_emb_prj_weight_sharing=args.tgt_emb_prj_weight_sharing, pe_maxlen=args.pe_maxlen) model = Transformer(encoder, decoder) # print(model) # model = nn.DataParallel(model) # optimizer optimizer = TransformerOptimizer( torch.optim.Adam(model.parameters(), betas=(0.9, 0.98), eps=1e-09)) else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 epochs_since_improvement = checkpoint['epochs_since_improvement'] model = checkpoint['model'] optimizer = checkpoint['optimizer'] # Move to GPU, if available model = model.to(device) # Custom dataloaders train_dataset = AiChallenger2017Dataset('train') train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, collate_fn=pad_collate, shuffle=True, num_workers=args.num_workers) valid_dataset = AiChallenger2017Dataset('valid') valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=args.batch_size, collate_fn=pad_collate, shuffle=False, num_workers=args.num_workers) # Epochs for epoch in range(start_epoch, args.epochs): # One epoch's training train_loss = train(train_loader=train_loader, model=model, optimizer=optimizer, epoch=epoch, logger=logger, writer=writer) writer.add_scalar('epoch/train_loss', train_loss, epoch) writer.add_scalar('epoch/learning_rate', optimizer.lr, epoch) print('\nLearning rate: {}'.format(optimizer.lr)) print('Step num: {}\n'.format(optimizer.step_num)) # One epoch's validation valid_loss = valid(valid_loader=valid_loader, model=model, logger=logger) writer.add_scalar('epoch/valid_loss', valid_loss, epoch) # Check if there was an improvement is_best = valid_loss < best_loss best_loss = min(valid_loss, best_loss) if not is_best: epochs_since_improvement += 1 print("\nEpochs since last improvement: %d\n" % (epochs_since_improvement, )) else: epochs_since_improvement = 0 # Save checkpoint save_checkpoint(epoch, epochs_since_improvement, model, optimizer, best_loss, is_best)
def main(): global char2index global index2char global SOS_token global EOS_token global PAD_token parser = argparse.ArgumentParser(description='Speech hackathon Baseline') parser.add_argument('--batch_size', type=int, default=32, help='batch size in training (default: 32)') parser.add_argument( '--workers', type=int, default=4, help='number of workers in dataset loader (default: 4)') parser.add_argument('--max_epochs', type=int, default=10, help='number of max epochs in training (default: 10)') parser.add_argument('--lr', type=float, default=0.0001, help='learning rate (default: 0.0001)') parser.add_argument('--teacher_forcing', type=float, default=0.5, help='teacher forcing ratio in decoder (default: 0.5)') parser.add_argument('--max_len', type=int, default=WORD_MAXLEN, help='maximum characters of sentence (default: 80)') parser.add_argument('--no_cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, help='random seed (default: 1)') parser.add_argument('--save_name', type=str, default='model', help='the name of model in nsml or local') parser.add_argument('--mode', type=str, default='train') parser.add_argument("--pause", type=int, default=0) parser.add_argument( '--word', action='store_true', help='Train/Predict model using word based label (default: False)') parser.add_argument('--gen_label_index', action='store_true', help='Generate word label index map(default: False)') parser.add_argument('--iteration', type=str, help='Iteratiom') parser.add_argument('--premodel_session', type=str, help='Session name of premodel') # transformer model parameter parser.add_argument('--d_model', type=int, default=128, help='transformer_d_model') parser.add_argument('--n_head', type=int, default=8, help='transformer_n_head') parser.add_argument('--num_encoder_layers', type=int, default=4, help='num_encoder_layers') parser.add_argument('--num_decoder_layers', type=int, default=4, help='transformer_num_decoder_layers') parser.add_argument('--dim_feedforward', type=int, default=2048, help='transformer_d_model') parser.add_argument('--dropout', type=float, default=0.1, help='transformer_dropout') # transformer warmup parameter parser.add_argument('--warmup_multiplier', type=int, default=3, help='transformer_warmup_multiplier') parser.add_argument('--warmup_epoch', type=int, default=10, help='transformer_warmup_epoch') args = parser.parse_args() char_loader = CharLabelLoader() char_loader.load_char2index('./hackathon.labels') label_loader = char_loader if args.word: if args.gen_label_index: generate_word_label_index_file(char_loader, TRAIN_LABEL_CHAR_PATH) from subprocess import call call(f'cat {TRAIN_LABEL_CHAR_PATH}', shell=True) # ??? ??? ??? ?? word_loader = CharLabelLoader() word_loader.load_char2index('./hackathon.pos.labels') label_loader = word_loader if os.path.exists(TRAIN_LABEL_CHAR_PATH): generate_word_label_file(char_loader, word_loader, TRAIN_LABEL_POS_PATH, TRAIN_LABEL_CHAR_PATH) char2index = label_loader.char2index index2char = label_loader.index2char SOS_token = char2index['<s>'] EOS_token = char2index['</s>'] PAD_token = char2index['_'] random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed_all(args.seed) args.cuda = not args.no_cuda and torch.cuda.is_available() device = torch.device('cuda' if args.cuda else 'cpu') ############ model print("model: transformer") # model = Transformer(d_model= args.d_model, n_head= args.n_head, num_encoder_layers= args.num_encoder_layers, num_decoder_layers= args.num_decoder_layers, # dim_feedforward= args.dim_feedforward, dropout= args.dropout, vocab_size= len(char2index), sound_maxlen= SOUND_MAXLEN, word_maxlen= WORD_MAXLEN) encoder = Encoder(d_input=128, n_layers=6, n_head=4, d_k=128, d_v=128, d_model=128, d_inner=2048, dropout=0.1, pe_maxlen=SOUND_MAXLEN) decoder = Decoder(sos_id=SOS_token, eos_id=EOS_token, n_tgt_vocab=len(char2index), d_word_vec=128, n_layers=6, n_head=4, d_k=128, d_v=128, d_model=128, d_inner=2048, dropout=0.1, tgt_emb_prj_weight_sharing=True, pe_maxlen=SOUND_MAXLEN) model = Transformer(encoder, decoder) optimizer = TransformerOptimizer( torch.optim.Adam(model.parameters(), lr=0.0004, betas=(0.9, 0.98), eps=1e-09)) ############/ for param in model.parameters(): param.data.uniform_(-0.08, 0.08) model = nn.DataParallel(model).to(device) """ optimizer = optim.Adam(model.module.parameters(), lr=args.lr) scheduler_cosine = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, args.max_epochs) scheduler_warmup = GradualWarmupScheduler(optimizer, multiplier=args.warmup_multiplier, total_epoch=args.warmup_epoch, after_scheduler=scheduler_cosine) criterion = nn.CrossEntropyLoss(reduction='sum', ignore_index=PAD_token).to(device) """ bind_model(model, optimizer) if args.pause == 1: nsml.paused(scope=locals()) if args.mode != "train": return data_list = os.path.join(DATASET_PATH, 'train_data', 'data_list.csv') wav_paths = list() script_paths = list() with open(data_list, 'r') as f: for line in f: # line: "aaa.wav,aaa.label" wav_path, script_path = line.strip().split(',') wav_paths.append(os.path.join(DATASET_PATH, 'train_data', wav_path)) script_paths.append( os.path.join(DATASET_PATH, 'train_data', script_path)) best_loss = 1e10 begin_epoch = 0 # load all target scripts for reducing disk i/o # target_path = os.path.join(DATASET_PATH, 'train_label') target_path = TRAIN_LABEL_CHAR_PATH if args.word: target_path = TRAIN_LABEL_POS_PATH load_targets(target_path) train_batch_num, train_dataset_list, valid_dataset = split_dataset( args, wav_paths, script_paths, valid_ratio=0.05) if args.iteration: if args.premodel_session: nsml.load(args.iteration, session=args.premodel_session) logger.info(f'Load {args.premodel_session} {args.iteration}') else: nsml.load(args.iteration) logger.info(f'Load {args.iteration}') logger.info('start') train_begin = time.time() for epoch in range(begin_epoch, args.max_epochs): # learning rate scheduler train_queue = queue.Queue(args.workers * 2) train_loader = MultiLoader(train_dataset_list, train_queue, args.batch_size, args.workers) train_loader.start() train_loss, train_cer = train(model, train_batch_num, train_queue, optimizer, device, train_begin, args.workers, 10, args.teacher_forcing) logger.info('Epoch %d (Training) Loss %0.4f CER %0.4f' % (epoch, train_loss, train_cer)) train_loader.join() print("~~~~~~~~~~~~") if epoch == 10 or (epoch > 48 and epoch % 10 == 9): valid_queue = queue.Queue(args.workers * 2) valid_loader = BaseDataLoader(valid_dataset, valid_queue, args.batch_size, 0) valid_loader.start() eval_loss, eval_cer = evaluate(model, valid_loader, valid_queue, device, args.max_len, args.batch_size) logger.info('Epoch %d (Evaluate) Loss %0.4f CER %0.4f' % (epoch, eval_loss, eval_cer)) valid_loader.join() nsml.report(False, step=epoch, train_epoch__loss=train_loss, train_epoch__cer=train_cer, eval__loss=eval_loss, eval__cer=eval_cer) best_model = (eval_loss < best_loss) nsml.save(args.save_name) if best_model: nsml.save('best') best_loss = eval_loss
def train_net(args): # 为了保证程序执行结果一致, 给随机化设定种子 torch.manual_seed(7) np.random.seed(7) checkpoint = args.checkpoint start_epoch = 0 writer = SummaryWriter() if checkpoint is None: # model encoder = Encoder(Config.vocab_size, args.n_layers_enc, args.n_head, args.d_k, args.d_v, args.d_model, args.d_inner, dropout=args.dropout, pe_maxlen=args.pe_maxlen) decoder = Decoder(Config.sos_id, Config.eos_id, Config.vocab_size, args.d_word_vec, args.n_layers_dec, args.n_head, args.d_k, args.d_v, args.d_model, args.d_inner, dropout=args.dropout, tgt_emb_prj_weight_sharing=args.tgt_emb_prj_weight_sharing, pe_maxlen=args.pe_maxlen) model = Transformer(encoder, decoder) # optimizer optimizer = TransformerOptimizer( torch.optim.Adam(model.parameters(), betas=(0.9, 0.98), eps=1e-09)) else: checkpoint = torch.load(checkpoint) start_epoch = checkpoint['epoch'] + 1 model = checkpoint['model'] optimizer = checkpoint['optimizer'] # Move to GPU, if available model = model.to(Config.device) # Custom dataloaders 数据的加载 注意这里指定了一个参数collate_fn代表的数据需要padding train_dataset = TranslateDataset() train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, collate_fn=pad_collate, shuffle=True, num_workers=args.num_workers) # Epochs Loss_list = [] for epoch in range(start_epoch, args.epochs): # One epoch's training train_loss = train(train_loader=train_loader, model=model, optimizer=optimizer, epoch=epoch, logger=logger, writer=writer) l = str(train_loss) Loss_list.append(l) l_temp = l + '\n' with open('loss_epoch.txt', 'a+') as f: f.write(l_temp) writer.add_scalar('epoch/train_loss', train_loss, epoch) writer.add_scalar('epoch/learning_rate', optimizer.lr, epoch) print('\nLearning rate: {}'.format(optimizer.lr)) print('Step num: {}\n'.format(optimizer.step_num)) # Save checkpoint save_checkpoint(epoch, model, optimizer, train_loss) with open('loss.txt', 'w') as f: f.write('\n'.join(Loss_list))
from transformer.optimizer import TransformerOptimizer from transformer.transformer import Transformer from utils import parse_args if __name__ == '__main__': k = 0.2 warmup_steps = 4000 init_lr = 512**(-0.5) args = parse_args() encoder = Encoder(args.d_input * args.LFR_m, args.n_layers_enc, args.n_head, args.d_k, args.d_v, args.d_model, args.d_inner, dropout=args.dropout, pe_maxlen=args.pe_maxlen) decoder = Decoder( sos_id, eos_id, vocab_size, args.d_word_vec, args.n_layers_dec, args.n_head, args.d_k, args.d_v, args.d_model, args.d_inner,
os.makedirs(LOG_PATH) eer_path = LOG_PATH / f_name res_path = LOG_PATH / r_name """ train """ # flat_shape = hp["numcep"] * hp["nb_time"] d_input = hp["numcep"] label_shape = len(train_speaker_list) # model d_m = hp["d_m"] encoder = Encoder(d_input=d_input, n_layers=2, d_k=d_m, d_v=d_m, d_m=d_m, d_ff=hp["d_ff"], dropout=0.1).to(device) pooling = SelfAttentionPooling(d_m, dropout=0.1).to(device) model = Transformer(encoder, pooling, d_m, label_shape, dropout=0.2).to(device) opt = torch.optim.Adam(model.parameters(), lr=hp["lr"], weight_decay=hp["weight_decay"]) loss_func = torch.nn.CrossEntropyLoss() best_eer = 99. if hp["comet"]: with experiment.train():
def __init__(self, params: dict): """ Instantiate the ``Transformer`` class. :param params: Dict containing the set of parameters for the entire model\ (e.g ``EncoderLayer``, ``DecoderLayer`` etc.) broken down in relevant sections, e.g.: params = { 'd_model': 512, 'src_vocab_size': 27000, 'tgt_vocab_size': 27000, 'N': 6, 'dropout': 0.1, 'attention': {'n_head': 8, 'd_k': 64, 'd_v': 64, 'dropout': 0.1}, 'feed-forward': {'d_ff': 2048, 'dropout': 0.1}, } """ # call base constructor super(Transformer, self).__init__() # Save params for Checkpoint self._params = params # instantiate Encoder layer enc_layer = EncoderLayer( size=params['d_model'], self_attention=MultiHeadAttention( n_head=params['attention']['n_head'], d_model=params['d_model'], d_k=params['attention']['d_k'], d_v=params['attention']['d_v'], dropout=params['attention']['dropout']), feed_forward=PositionwiseFeedForward( d_model=params['d_model'], d_ff=params['feed-forward']['d_ff'], dropout=params['feed-forward']['dropout']), dropout=params['dropout']) # instantiate Encoder self.encoder = Encoder(layer=enc_layer, n_layers=params['N']) # instantiate Decoder layer decoder_layer = DecoderLayer( size=params['d_model'], self_attn=MultiHeadAttention( n_head=params['attention']['n_head'], d_model=params['d_model'], d_k=params['attention']['d_k'], d_v=params['attention']['d_v'], dropout=params['attention']['dropout']), memory_attn=MultiHeadAttention( n_head=params['attention']['n_head'], d_model=params['d_model'], d_k=params['attention']['d_k'], d_v=params['attention']['d_v'], dropout=params['attention']['dropout']), feed_forward=PositionwiseFeedForward( d_model=params['d_model'], d_ff=params['feed-forward']['d_ff'], dropout=params['feed-forward']['dropout']), dropout=params['dropout']) # instantiate Decoder self.decoder = Decoder(layer=decoder_layer, N=params['N']) pos_encoding = PositionalEncoding(d_model=params['d_model'], dropout=params['dropout']) self.src_embeddings = nn.Sequential( Embeddings(d_model=params['d_model'], vocab_size=params['src_vocab_size']), pos_encoding) self.trg_embeddings = nn.Sequential( Embeddings(d_model=params['d_model'], vocab_size=params['tgt_vocab_size']), pos_encoding) self.classifier = OutputClassifier(d_model=params['d_model'], vocab=params['tgt_vocab_size']) # Initialize parameters with Glorot / fan_avg. for p in self.parameters(): if p.dim() > 1: nn.init.xavier_uniform_(p)
def __init__(self, n_src_vocab, n_tgt_vocab, len_max_seq_enc, len_max_seq_dec, d_word_vec=512, d_model=512, d_inner=2048, n_layers=6, n_head=8, d_k=64, d_v=64, dropout=0.1, tgt_emb_prj_weight_sharing=True, emb_src_tgt_weight_sharing=True, pretrained_embeddings=None): super().__init__() self.encoder = Encoder(n_src_vocab=n_src_vocab, len_max_seq=len_max_seq_enc, d_word_vec=d_word_vec, d_model=d_model, d_inner=d_inner, n_layers=n_layers, n_head=n_head, d_k=d_k, d_v=d_v, dropout=dropout, pretrained_embeddings=pretrained_embeddings) self.decoder = Decoder(n_tgt_vocab=n_tgt_vocab, len_max_seq=len_max_seq_dec, d_word_vec=d_word_vec, d_model=d_model, d_inner=d_inner, n_layers=n_layers, n_head=n_head, d_k=d_k, d_v=d_v, dropout=dropout, pretrained_embeddings=pretrained_embeddings) self.tgt_word_prj = nn.Linear(d_model, n_tgt_vocab, bias=False) nn.init.xavier_normal_(self.tgt_word_prj.weight) assert d_model == d_word_vec, \ 'To facilitate the residual connections, \ the dimensions of all module outputs shall be the same.' if tgt_emb_prj_weight_sharing: # Share the weight matrix between target word embedding & the final logit dense layer self.tgt_word_prj.weight = self.decoder.tgt_word_emb.weight self.x_logit_scale = (d_model**-0.5) else: self.x_logit_scale = 1. if emb_src_tgt_weight_sharing: # Share the weight matrix between source & target word embeddings assert n_src_vocab == n_tgt_vocab, \ "To share word embedding table, the vocabulary size of src/tgt shall be the same." self.encoder.src_word_emb.weight = self.decoder.tgt_word_emb.weight