def __init__(self,
masking_mode: str = None,
token_indexers: Dict[str, TokenIndexer] = None,
lazy: bool = False) -> None:
super().__init__(lazy)
if masking_mode and masking_mode.lower() not in [
'ner_least_specific', 'ner_most_specific'
]:
raise ValueError(f"Masking mode '{masking_mode}' not supported.")
self._masking_mode = masking_mode
self._token_splitter = OpenAISplitter()
self._token_indexers = token_indexers or {
'tokens': SingleIdTokenIndexer()
}
def setUp(self):
super(TestOpenAiWordSplitter, self).setUp()
self.word_splitter = OpenAISplitter()
def setUp(self):
super().setUp()
self.word_splitter = OpenAISplitter()