def initialize_model(): config = get_config() device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu') #device = torch.device('cpu') print("device", device) '''create tokenizers''' tokenizer = ByteLevelBPETokenizer( "data/english_tokenizer-vocab.json", "data/english_tokenizer-merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_padding(pad_token='[PAD]', length=config['max_len']) tokenizer.enable_truncation(max_length=config['max_len']) ''' Create model ''' vocab_size = len(tokenizer.get_vocab()) print("tokenizer.vocab_size", vocab_size) model = TransformerModel(config['embedding_size'], vocab_size, vocab_size, config['src_pad_idx'], config['num_heads'], config['num_encoder_layers'], config['num_decoder_layers'], config['forward_expansion'], config['dropout'], config['max_len'], device) checkpoint = torch.load(config['pretrained_model'], map_location=device) model.load_state_dict(checkpoint['net']) model.eval() model = model.to(device) return config, model, tokenizer, device
def create_norwegian_tokenizer(): tokenizer = ByteLevelBPETokenizer( "./models/KariBERTa-tiny/vocab.json", "./models/KariBERTa-tiny/merges.txt", ) tokenizer._tokenizer.post_processor = BertProcessing( ("</s>", tokenizer.token_to_id("</s>")), ("<s>", tokenizer.token_to_id("<s>")), ) tokenizer.enable_truncation(max_length=512) tokenizer.enable_padding() return tokenizer