class InputConfig(ConfigBase):
right_tokens: RoBERTaTensorizer.Config = RoBERTaTensorizer.Config()
left_tokens: RoBERTaTensorizer.Config = RoBERTaTensorizer.Config()
right_dense: Optional[FloatListTensorizer.Config] = None
left_dense: Optional[FloatListTensorizer.Config] = None
labels: LabelTensorizer.Config = LabelTensorizer.Config()
def test_roberta_tensorizer(self):
text = "Prototype"
tokens = [[0, 4, 5, 2]]
pad_masks = [[1, 1, 1, 1]]
segment_labels = [[0, 0, 0, 0]]
positions = [[0, 1, 2, 3]]
expected = [tokens, pad_masks, segment_labels, positions]
tensorizer = RoBERTaTensorizer.from_config(
RoBERTaTensorizer.Config(
tokenizer=GPT2BPETokenizer.Config(
bpe_encoder_path="pytext/data/test/data/gpt2_encoder.json",
bpe_vocab_path="pytext/data/test/data/gpt2_vocab.bpe",
),
vocab_file="pytext/data/test/data/gpt2_dict.txt",
max_seq_len=256,
))
tensors = tensorizer.tensorize([tensorizer.numberize({"text": text})])
for tensor, expect in zip(tensors, expected):
self.assertEqual(tensor.tolist(), expect)
tensorizer_impl = RoBERTaTensorizerScriptImpl(
tokenizer=DoNothingTokenizer(),
vocab=tensorizer.vocab,
max_seq_len=tensorizer.max_seq_len,
).torchscriptify()
per_sentence_tokens = [tensorizer.tokenizer.tokenize(text)]
tokens_2d, segment_labels_2d, seq_lens_1d, positions_2d = zip(
*[tensorizer_impl.numberize(per_sentence_tokens)])
script_tensors = tensorizer_impl.tensorize(tokens_2d,
segment_labels_2d,
seq_lens_1d, positions_2d)
for tensor, expect in zip(script_tensors, expected):
self.assertEqual(tensor.tolist(), expect)
def __init__(
self,
columns: List[str] = Config.columns,
vocab: Optional[Vocabulary] = None,
tokenizer: Optional[Tokenizer] = None,
max_seq_len: int = Config.max_seq_len,
):
RoBERTaTensorizer.__init__(
self,
columns=columns,
vocab=vocab,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
)
def __init__(
self,
columns: List[str] = Config.columns,
vocab: Vocabulary = None,
tokenizer: Tokenizer = None,
max_seq_len: int = Config.max_seq_len,
answers_column: str = Config.answers_column,
answer_starts_column: str = Config.answer_starts_column,
):
RoBERTaTensorizer.__init__(
self,
columns=columns,
vocab=vocab,
tokenizer=tokenizer,
max_seq_len=max_seq_len,
)
self.answers_column = answers_column
self.answer_starts_column = answer_starts_column
self.wrap_special_tokens = False
class RegressionModelInput(ConfigBase):
tokens: RoBERTaTensorizer.Config = RoBERTaTensorizer.Config()
labels: NumericLabelTensorizer.Config = NumericLabelTensorizer.Config(
)
def _lookup_tokens(self, text):
return RoBERTaTensorizer._lookup_tokens(self, text)
class InputConfig(ConfigBase):
tokens: RoBERTaTensorizer.Config = RoBERTaTensorizer.Config()
right_dense: FloatListTensorizer.Config = None
left_dense: FloatListTensorizer.Config = None
labels: LabelTensorizer.Config = LabelTensorizer.Config()
class InputConfig(ConfigBase):
tokens: RoBERTaTensorizer.Config = RoBERTaTensorizer.Config()
labels: LabelTensorizer.Config = LabelTensorizer.Config()