def _get_indexer(self, weights_name, special_tokens_fix):
bert_token_indexer = PretrainedBertIndexer(
pretrained_model=weights_name,
do_lowercase=self.lowercase_tokens,
max_pieces_per_token=5,
use_starting_offsets=True,
truncate_long_sequences=True,
special_tokens_fix=special_tokens_fix,
is_test=True)
return {'bert': bert_token_indexer}
def get_token_indexers(model_name, max_pieces_per_token=5, lowercase_tokens=True, special_tokens_fix=0, is_test=False):
bert_token_indexer = PretrainedBertIndexer(
pretrained_model=model_name,
max_pieces_per_token=max_pieces_per_token,
do_lowercase=lowercase_tokens,
use_starting_offsets=True,
special_tokens_fix=special_tokens_fix,
is_test=is_test
)
return {'bert': bert_token_indexer}