def _get_indexer(self, weights_name, special_tokens_fix): bert_token_indexer = PretrainedBertIndexer( pretrained_model=weights_name, do_lowercase=self.lowercase_tokens, max_pieces_per_token=5, use_starting_offsets=True, truncate_long_sequences=True, special_tokens_fix=special_tokens_fix, is_test=True) return {'bert': bert_token_indexer}
def get_token_indexers(model_name, max_pieces_per_token=5, lowercase_tokens=True, special_tokens_fix=0, is_test=False): bert_token_indexer = PretrainedBertIndexer( pretrained_model=model_name, max_pieces_per_token=max_pieces_per_token, do_lowercase=lowercase_tokens, use_starting_offsets=True, special_tokens_fix=special_tokens_fix, is_test=is_test ) return {'bert': bert_token_indexer}