def initialize(self, resources: Resources, configs: Config):
self.resources = resources
self.config = Config(configs, self.default_configs())
# TODO: At the time of writing, no way in texar to set encoder in
# `texar.torch.modules.classifiers.BertClassifier`. Should not ideally
# be changing a private variable.
# pylint: disable=protected-access
BERTClassifier._ENCODER_CLASS = BERTEncoder
# pylint: enable=protected-access
cache_dir = os.path.join(os.path.dirname(__file__),
self.config.model_dir)
self.device = torch.device('cuda:0') \
if torch.cuda.is_available() else torch.device('cpu')
self.model = BERTClassifier(
pretrained_model_name=self.config.pretrained_model_name,
cache_dir=cache_dir,
hparams=self.config).to(self.device)
self.tokenizer = BERTTokenizer(
pretrained_model_name=self.config.pretrained_model_name,
cache_dir=cache_dir,
hparams=None)
def initialize(self, resources: Resources, configs: Config):
super().initialize(resources, configs)
if not self.configs.tokenizer_configs.pretrained_model_name:
raise ValueError("Please specify a pretrained bert model")
self.tokenizer = BERTTokenizer(
cache_dir=None,
hparams=self.configs.tokenizer_configs,
)
self.aligner = DiffAligner()
self.__do_lower_case = self.configs.tokenizer_configs.do_lower_case
def test_encode_text(self):
tokenizer = BERTTokenizer.load(self.vocab_file)
text_1 = u"He is very happy"
text_2 = u"unwanted, running"
text_1_ids = tokenizer.map_text_to_id(text_1)
text_2_ids = tokenizer.map_text_to_id(text_2)
cls_token_id = tokenizer.map_token_to_id(tokenizer.cls_token)
sep_token_id = tokenizer.map_token_to_id(tokenizer.sep_token)
input_ids, segment_ids, input_mask = \
tokenizer.encode_text(text_1, None, 4)
self.assertListEqual(input_ids,
[cls_token_id] + text_1_ids[:2] + [sep_token_id])
self.assertListEqual(segment_ids, [0, 0, 0, 0])
self.assertListEqual(input_mask, [1, 1, 1, 1])
input_ids, segment_ids, input_mask = \
tokenizer.encode_text(text_1, text_2, 7)
self.assertListEqual(input_ids, [cls_token_id] + text_1_ids[:2] +
[sep_token_id] + text_2_ids[:2] + [sep_token_id])
self.assertListEqual(segment_ids, [0, 0, 0, 0, 1, 1, 1])
self.assertListEqual(input_mask, [1, 1, 1, 1, 1, 1, 1])
def default_configs(cls):
"""Returns the configuration with default values.
Here:
- `tokenizer_configs` contains all default
hyper-parameters in
:class:`~texar.torch.data.tokenizer.bert_tokenizer.BERTTokenizer`,
this processor will pass on all the configurations to the
tokenizer to create the tokenizer instance.
- `segment_unit` contains an Annotation entry type used to split the
text into smaller units. For example, setting this to
`ft.onto.base_ontology.Sentence` will make this tokenizer do
tokenization on a sentence base, which could be more efficient
when the alignment is used.
- `token_source` contains entry name of where the tokens come from.
For example, setting this to `ft.onto.base_ontology.Token` will
make this tokenizer split the sub-word based on this token. The
default value will use `ft.onto.base_ontology.Token`. If this
value is set to None, then it will use `word_tokenization`
function of this class to do tokenization.
Note that if `segment_unit` or `token_source` is provided, the
:meth:`~forte.processors.base.base_processor.BaseProcessor.check_record`
will check if certain types are written before this processor.
Returns: Default configuration value for the tokenizer.
"""
return {
"tokenizer_configs": BERTTokenizer.default_hparams(),
"segment_unit": None,
"token_source": "ft.onto.base_ontology.Token",
"@no_typecheck": "token_source",
}
def test_tokenize(self):
tokenizer = BERTTokenizer.load(self.vocab_file)
tokens = tokenizer.map_text_to_token(u"UNwant\u00E9d,running")
self.assertListEqual(tokens,
["un", "##want", "##ed", ",", "runn", "##ing"])
ids = tokenizer.map_token_to_id(tokens)
self.assertListEqual(ids, [7, 4, 5, 10, 8, 9])
def initialize(self, resources: Resources, configs: Config):
super().initialize(resources, configs)
self.resources = resources
self.config = Config(configs, self.default_configs())
if not self.config.pretrained_model_name:
raise ValueError("Please specify a pretrained bert model")
self.tokenizer = BERTTokenizer(
pretrained_model_name=self.config.pretrained_model_name,
cache_dir=None,
hparams=None,
)
def test_save_load(self):
tokenizer = BERTTokenizer.load(self.vocab_file)
before_tokens = tokenizer.map_text_to_id(
u"He is very happy, UNwant\u00E9d,running")
with tempfile.TemporaryDirectory() as tmpdirname:
tokenizer.save(tmpdirname)
tokenizer = tokenizer.load(tmpdirname)
after_tokens = tokenizer.map_text_to_id(
u"He is very happy, UNwant\u00E9d,running")
self.assertListEqual(before_tokens, after_tokens)
def test_add_tokens(self):
tokenizer = BERTTokenizer.load(self.vocab_file)
vocab_size = tokenizer.vocab_size
all_size = len(tokenizer)
self.assertNotEqual(vocab_size, 0)
self.assertEqual(vocab_size, all_size)
new_toks = ["aaaaabbbbbb", "cccccccccdddddddd"]
added_toks = tokenizer.add_tokens(new_toks)
vocab_size_2 = tokenizer.vocab_size
all_size_2 = len(tokenizer)
self.assertNotEqual(vocab_size_2, 0)
self.assertEqual(vocab_size, vocab_size_2)
self.assertEqual(added_toks, len(new_toks))
self.assertEqual(all_size_2, all_size + len(new_toks))
tokens = tokenizer.map_text_to_id(
"aaaaabbbbbb low cccccccccdddddddd l")
self.assertGreaterEqual(len(tokens), 4)
self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
new_toks_2 = {
'eos_token': ">>>>|||<||<<|<<",
'pad_token': "<<<<<|||>|>>>>|>"
}
added_toks_2 = tokenizer.add_special_tokens(new_toks_2)
vocab_size_3 = tokenizer.vocab_size
all_size_3 = len(tokenizer)
self.assertNotEqual(vocab_size_3, 0)
self.assertEqual(vocab_size, vocab_size_3)
self.assertEqual(added_toks_2, len(new_toks_2))
self.assertEqual(all_size_3, all_size_2 + len(new_toks_2))
tokens = tokenizer.map_text_to_id(
">>>>|||<||<<|<< aaaaabbbbbb low cccccccccdddddddd "
"<<<<<|||>|>>>>|> l")
self.assertGreaterEqual(len(tokens), 6)
self.assertGreater(tokens[0], tokenizer.vocab_size - 1)
self.assertGreater(tokens[0], tokens[1])
self.assertGreater(tokens[-2], tokenizer.vocab_size - 1)
self.assertGreater(tokens[-2], tokens[-3])
self.assertEqual(tokens[0],
tokenizer.map_token_to_id(tokenizer.eos_token))
self.assertEqual(tokens[-2],
tokenizer.map_token_to_id(tokenizer.pad_token))
def initialize(self, config: Union[Dict, Config]):
# pylint: disable=attribute-defined-outside-init
super().initialize(config=config)
self.tokenizer = BERTTokenizer(
pretrained_model_name=self.config.pretrained_model_name,
cache_dir=None,
hparams=None,
)
predefined_dict = [key for key, _ in self.tokenizer.vocab.items()]
self.predefined_vocab(predefined_dict)
if not self.vocab:
raise AttributeError("Vocabulary is required in SubwordExtractor.")
self.vocab.mark_special_element(self.tokenizer.vocab["[PAD]"], "PAD")
self.vocab.mark_special_element(self.tokenizer.vocab["[UNK]"], "UNK")
def test_pickle(self):
tokenizer = BERTTokenizer.load(self.vocab_file)
self.assertIsNotNone(tokenizer)
text = u"Munich and Berlin are nice cities"
subwords = tokenizer.map_text_to_token(text)
with tempfile.TemporaryDirectory() as tmpdirname:
filename = os.path.join(tmpdirname, u"tokenizer.bin")
with open(filename, "wb") as f:
pickle.dump(tokenizer, f)
with open(filename, "rb") as f:
tokenizer_new = pickle.load(f)
subwords_loaded = tokenizer_new.map_text_to_token(text)
self.assertListEqual(subwords, subwords_loaded)
def test_encode_decode(self):
tokenizer = BERTTokenizer.load(self.vocab_file)
input_text = u"UNwant\u00E9d,running"
output_text = u"unwanted, running"
tokens = tokenizer.map_text_to_token(input_text)
ids = tokenizer.map_token_to_id(tokens)
ids_2 = tokenizer.map_text_to_id(input_text)
self.assertListEqual(ids, ids_2)
tokens_2 = tokenizer.map_id_to_token(ids)
text_2 = tokenizer.map_id_to_text(ids)
self.assertEqual(text_2, output_text)
self.assertNotEqual(len(tokens_2), 0)
self.assertIsInstance(text_2, str)
def test_model_loading(self):
for pretrained_model_name in BERTTokenizer.available_checkpoints():
tokenizer = BERTTokenizer(
pretrained_model_name=pretrained_model_name)
_ = tokenizer.map_text_to_token(u"UNwant\u00E9d,running")
class BertRerankingProcessor(MultiPackProcessor):
def initialize(self, resources: Resources, configs: Config):
self.resources = resources
self.config = Config(configs, self.default_configs())
# TODO: At the time of writing, no way in texar to set encoder in
# `texar.torch.modules.classifiers.BertClassifier`. Should not ideally
# be changing a private variable.
# pylint: disable=protected-access
BERTClassifier._ENCODER_CLASS = BERTEncoder
# pylint: enable=protected-access
cache_dir = os.path.join(os.path.dirname(__file__),
self.config.model_dir)
self.device = torch.device('cuda:0') \
if torch.cuda.is_available() else torch.device('cpu')
self.model = BERTClassifier(
pretrained_model_name=self.config.pretrained_model_name,
cache_dir=cache_dir,
hparams=self.config).to(self.device)
self.tokenizer = BERTTokenizer(
pretrained_model_name=self.config.pretrained_model_name,
cache_dir=cache_dir,
hparams=None)
@classmethod
def default_configs(cls) -> Dict[str, Any]:
configs = super().default_configs()
pretrained_model_name = "bert-large-uncased"
configs.update({
"size":
5,
"query_pack_name":
"query",
"field":
"content",
"pretrained_model_name":
pretrained_model_name,
"model_dir":
os.path.join(os.path.dirname(__file__), "models"),
"max_seq_length":
512
})
return configs
def _process(self, input_pack: MultiPack):
max_len = self.config.max_seq_length
query_pack_name = self.config.query_pack_name
query_pack = input_pack.get_pack(self.config.query_pack_name)
query_entry = list(query_pack.get(Query))[0]
query_text = query_pack.text
packs = {}
for doc_id in input_pack.pack_names:
if doc_id == query_pack_name:
continue
pack = input_pack.get_pack(doc_id)
document_text = pack.text
# BERT Inference
input_ids, segment_ids, input_mask = [
torch.LongTensor(item).unsqueeze(0).to(self.device) for item in
self.tokenizer.encode_text(query_text, document_text, max_len)
]
seq_length = (input_mask == 1).sum(dim=-1)
logits, _ = self.model(input_ids, seq_length, segment_ids)
preds = torch.nn.functional.softmax(torch.Tensor(logits), dim=1)
score = preds.detach().tolist()[0][1]
query_entry.update_results({doc_id: score})
packs[doc_id] = pack
class SubwordTokenizer(PackProcessor):
"""
Subword Tokenizer using pretrained Bert model.
"""
def __init__(self):
super().__init__()
self.tokenizer: BERTTokenizer = None
self.aligner: DiffAligner = None
self.__do_lower_case = True
# pylint: disable=attribute-defined-outside-init,unused-argument
def initialize(self, resources: Resources, configs: Config):
super().initialize(resources, configs)
if not self.configs.tokenizer_configs.pretrained_model_name:
raise ValueError("Please specify a pretrained bert model")
self.tokenizer = BERTTokenizer(
cache_dir=None,
hparams=self.configs.tokenizer_configs,
)
self.aligner = DiffAligner()
self.__do_lower_case = self.configs.tokenizer_configs.do_lower_case
def _process(self, input_pack: DataPack):
assert self.tokenizer is not None
assert self.aligner is not None
if self.configs.token_source is not None:
# Use provided token source.
token: Annotation
for token in input_pack.get(self.configs.token_source):
assert isinstance(token, Annotation)
self.__add_subwords(
input_pack,
token.text # type: ignore
if self.__do_lower_case else token.text, # type: ignore
token.begin, # type: ignore
)
elif self.configs.segment_unit is not None:
# If token source not provide, try to use provided segments.
segment: Annotation
for segment in input_pack.get(self.configs.segment_unit):
self._segment(
input_pack,
segment.text,
segment.begin # type: ignore
)
else:
# Use the whole data pack, maybe less efficient in some cases.
self._segment(input_pack, input_pack.text, 0)
def _word_tokenization(self,
text: str) -> Iterator[Tuple[str, Tuple[int, int]]]:
"""
This function should tokenize the text and return the tokenization
results in the form of a word and the span of each word. A span is the
begin and end of this word, indexed from 0, and end = begin + length
of the word.
By default, this calls the Texar's BasicTokenizer and then align the
result back. You can implement this function if you prefer a
different tokenizer.
Args:
text: Input text to be tokenized.
Returns: A iterator of tokenization result in the form of triplets of
(word, (begin, end)).
"""
basic_tokens: List[str] = self.tokenizer.basic_tokenizer.tokenize(
text, never_split=self.tokenizer.all_special_tokens)
token_spans = self.aligner.align_with_segments(text, basic_tokens)
for t, span in zip(basic_tokens, token_spans):
if span is not None:
yield t, span
def _segment(self, pack: DataPack, text: str, segment_offset: int):
if self.tokenizer.do_basic_tokenize:
for token, (token_begin, _) in self._word_tokenization(text):
assert token is not None
self.__add_subwords(pack, text, token_begin + segment_offset)
else:
self.__add_subwords(pack, text, segment_offset)
def __add_subwords(self, pack: DataPack, text: str, text_offset: int):
if self.__do_lower_case:
lower_text = text.lower()
# See this https://bugs.python.org/issue17252 to understand why this
# is checked here. tl;dr sometimes lower casing special unicode
# string will result in a change of length due to unicode NFD.
if len(lower_text) == len(text):
text = text.lower()
for (
subword,
begin,
end,
) in self.tokenizer.wordpiece_tokenizer.tokenize_with_span(text):
subword_token = Subword(pack, begin + text_offset,
end + text_offset)
if subword == self.tokenizer.wordpiece_tokenizer.unk_token:
subword_token.is_unk = True
subword_token.is_first_segment = not subword.startswith("##")
# pylint: disable=protected-access
subword_token.vocab_id = self.tokenizer._map_token_to_id(subword)
def record(self, record_meta: Dict[str, Set[str]]):
r"""Method to add output type record of current processor
to :attr:`forte.data.data_pack.Meta.record`.
Args:
record_meta: the field in the data pack storing type records needed
in for consistency checking.
Returns:
"""
record_meta["ft.onto.base_ontology.Subword"] = {
"is_unk",
"is_first_segment",
"vocab_id",
}
def expected_types_and_attributes(self) -> Dict[str, Set[str]]:
r"""Method to add expected type for current processor input which
would be checked before running the processor if
the pipeline is initialized with
`enforce_consistency=True` or
:meth:`~forte.pipeline.Pipeline.enforce_consistency` was enabled for
the pipeline.
"""
expected_types: Dict[str, Set[str]] = {}
if self.configs.token_source is not None:
expected_types[self.configs.token_source] = set()
elif self.configs.segment_unit is not None:
expected_types[self.configs.segment_unit] = set()
return expected_types
@classmethod
def default_configs(cls):
"""Returns the configuration with default values.
Here:
- `tokenizer_configs` contains all default
hyper-parameters in
:class:`~texar.torch.data.tokenizer.bert_tokenizer.BERTTokenizer`,
this processor will pass on all the configurations to the
tokenizer to create the tokenizer instance.
- `segment_unit` contains an Annotation entry type used to split the
text into smaller units. For example, setting this to
`ft.onto.base_ontology.Sentence` will make this tokenizer do
tokenization on a sentence base, which could be more efficient
when the alignment is used.
- `token_source` contains entry name of where the tokens come from.
For example, setting this to `ft.onto.base_ontology.Token` will
make this tokenizer split the sub-word based on this token. The
default value will use `ft.onto.base_ontology.Token`. If this
value is set to None, then it will use `word_tokenization`
function of this class to do tokenization.
Note that if `segment_unit` or `token_source` is provided, the
:meth:`~forte.processors.base.base_processor.BaseProcessor.check_record`
will check if certain types are written before this processor.
Returns: Default configuration value for the tokenizer.
"""
return {
"tokenizer_configs": BERTTokenizer.default_hparams(),
"segment_unit": None,
"token_source": "ft.onto.base_ontology.Token",
"@no_typecheck": "token_source",
}