def initialize_model():
config = get_config()
device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')
#device = torch.device('cpu')
print("device", device)
'''create tokenizers'''
tokenizer = ByteLevelBPETokenizer(
"data/english_tokenizer-vocab.json",
"data/english_tokenizer-merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_padding(pad_token='[PAD]', length=config['max_len'])
tokenizer.enable_truncation(max_length=config['max_len'])
'''
Create model
'''
vocab_size = len(tokenizer.get_vocab())
print("tokenizer.vocab_size", vocab_size)
model = TransformerModel(config['embedding_size'], vocab_size, vocab_size,
config['src_pad_idx'], config['num_heads'],
config['num_encoder_layers'],
config['num_decoder_layers'],
config['forward_expansion'], config['dropout'],
config['max_len'], device)
checkpoint = torch.load(config['pretrained_model'], map_location=device)
model.load_state_dict(checkpoint['net'])
model.eval()
model = model.to(device)
return config, model, tokenizer, device
def create_norwegian_tokenizer():
tokenizer = ByteLevelBPETokenizer(
"./models/KariBERTa-tiny/vocab.json",
"./models/KariBERTa-tiny/merges.txt",
)
tokenizer._tokenizer.post_processor = BertProcessing(
("</s>", tokenizer.token_to_id("</s>")),
("<s>", tokenizer.token_to_id("<s>")),
)
tokenizer.enable_truncation(max_length=512)
tokenizer.enable_padding()
return tokenizer