class TSVTaggingTransform(TsvTaggingFormat, Transform):
def __init__(self,
config: SerializableDict = None,
map_x=True,
map_y=True,
use_char=False,
**kwargs) -> None:
super().__init__(**merge_locals_kwargs(locals(), kwargs))
self.word_vocab: Optional[VocabTF] = None
self.tag_vocab: Optional[VocabTF] = None
self.char_vocab: Optional[VocabTF] = None
def fit(self, trn_path: str, **kwargs) -> int:
self.word_vocab = VocabTF()
self.tag_vocab = VocabTF(pad_token=None, unk_token=None)
num_samples = 0
for words, tags in self.file_to_inputs(trn_path, True):
self.word_vocab.update(words)
self.tag_vocab.update(tags)
num_samples += 1
if self.char_vocab:
self.char_vocab = VocabTF()
for word in self.word_vocab.token_to_idx.keys():
if word in (self.word_vocab.pad_token,
self.word_vocab.unk_token):
continue
self.char_vocab.update(list(word))
return num_samples
def create_types_shapes_values(self) -> Tuple[Tuple, Tuple, Tuple]:
types = tf.string, tf.string
shapes = [None], [None]
values = self.word_vocab.pad_token, self.tag_vocab.first_token
return types, shapes, values
def inputs_to_samples(self, inputs, gold=False):
lower = self.config.get('lower', False)
if gold:
if lower:
for x, y in inputs:
yield x.lower(), y
else:
yield from inputs
else:
for x in inputs:
yield x.lower() if lower else x, [self.padding_values[-1]
] * len(x)
def x_to_idx(self, x) -> Union[tf.Tensor, Tuple]:
return self.word_vocab.lookup(x)
def y_to_idx(self, y) -> tf.Tensor:
return self.tag_vocab.lookup(y)
def X_to_inputs(self, X: Union[tf.Tensor, Tuple[tf.Tensor]]) -> Iterable:
for xs in X:
words = []
for x in xs:
words.append(
str_tensor_to_str(x) if self.char_vocab else self.
word_vocab.idx_to_token[int(x)])
yield words
def Y_to_outputs(self,
Y: Union[tf.Tensor, Tuple[tf.Tensor]],
gold=False,
inputs=None,
X=None,
**kwargs) -> Iterable:
if not gold:
Y = tf.argmax(Y, axis=2)
for ys, xs in zip(Y, inputs):
tags = []
for y, x in zip(ys, xs):
tags.append(self.tag_vocab.idx_to_token[int(y)])
yield tags
def input_is_single_sample(
self, input: Union[List[str], List[List[str]]]) -> bool:
return isinstance(input[0], str)
def input_truth_output_to_str(self, input: List[str], truth: List[str],
output: List[str]):
text = ''
for word, gold_tag, pred_tag in zip(input, truth, output):
text += ' '.join([word, gold_tag, pred_tag]) + '\n'
text += '\n'
return text
class TextTransform(Transform):
def __init__(self,
forward=True,
seq_len=10,
tokenizer='char',
config: SerializableDict = None,
map_x=True,
map_y=True,
**kwargs) -> None:
super().__init__(config,
map_x,
map_y,
seq_len=seq_len,
tokenizer=tokenizer,
forward=forward,
**kwargs)
self.vocab: VocabTF = None
def tokenize_func(self):
if self.config.tokenizer == 'char':
return list
elif self.config.tokenizer == 'whitespace':
return lambda x: x.split()
else:
return lambda x: x.split(self.config.tokenizer)
def fit(self, trn_path: str, **kwargs) -> int:
self.vocab = VocabTF()
num_samples = 0
for x, y in self.file_to_inputs(trn_path):
self.vocab.update(x)
num_samples += 1
return num_samples
def create_types_shapes_values(self) -> Tuple[Tuple, Tuple, Tuple]:
types = tf.string, tf.string
shapes = [None], [None]
defaults = self.vocab.pad_token, self.vocab.pad_token
return types, shapes, defaults
def file_to_inputs(self, filepath: str, gold=True):
forward = self.config.forward
seq_len = self.config.seq_len
buffer = []
tokenizer = self.tokenize_func()
with open(filepath,
encoding='utf-8') if forward else FileReadBackwards(
filepath, encoding="utf-8") as src:
for line in src:
tokens = tokenizer(line)
buffer += tokens
while len(buffer) > seq_len:
yield buffer[:seq_len], buffer[1:1 + seq_len]
buffer.pop(0)
def inputs_to_samples(self, inputs, gold=False):
forward = self.config.forward
for t in inputs:
if gold:
x, y = t
else:
x, y = t, t
if not forward:
x = list(reversed(x))
y = list(reversed(y))
yield x, y
def x_to_idx(self, x) -> Union[tf.Tensor, Tuple]:
return self.vocab.lookup(x)
def y_to_idx(self, y) -> tf.Tensor:
return self.x_to_idx(y)
def Y_to_outputs(self,
Y: Union[tf.Tensor, Tuple[tf.Tensor]],
gold=False,
inputs=None,
**kwargs) -> Iterable:
pred = tf.argmax(Y, axis=-1)
for ys, ms in zip(pred, inputs):
ret = []
for y in ys:
ret.append(self.vocab.idx_to_token[int(y)])
yield ret
def input_is_single_sample(self, input: Any) -> bool:
return isinstance(input[0], str)
class TACREDTransform(Transform):
def __init__(self,
config: SerializableDict = None,
map_x=True,
map_y=True,
lower=False,
**kwargs) -> None:
super().__init__(**merge_locals_kwargs(locals(), kwargs))
self.token_vocab = VocabTF()
self.pos_vocab = VocabTF(pad_token=None, unk_token=None)
self.ner_vocab = VocabTF(pad_token=None)
self.deprel_vocab = VocabTF(pad_token=None, unk_token=None)
self.rel_vocab = VocabTF(pad_token=None, unk_token=None)
def fit(self, trn_path: str, **kwargs) -> int:
count = 0
for (tokens, pos, ner, head, deprel, subj_positions, obj_positions,
subj_type, obj_type), relation in self.file_to_samples(trn_path,
gold=True):
count += 1
self.token_vocab.update(tokens)
self.pos_vocab.update(pos)
self.ner_vocab.update(ner)
self.deprel_vocab.update(deprel)
self.rel_vocab.add(relation)
return count
def file_to_inputs(self, filepath: str, gold=True):
data = load_json(filepath)
for d in data:
tokens = list(d['token'])
ss, se = d['subj_start'], d['subj_end']
os, oe = d['obj_start'], d['obj_end']
pos = d['stanford_pos']
ner = d['stanford_ner']
deprel = d['stanford_deprel']
head = [int(x) for x in d['stanford_head']]
assert any([x == 0 for x in head])
relation = d['relation']
yield (tokens, pos, ner, head, deprel, ss, se, os, oe), relation
def inputs_to_samples(self, inputs, gold=False):
for input in inputs:
if gold:
(tokens, pos, ner, head, deprel, ss, se, os,
oe), relation = input
else:
tokens, pos, ner, head, deprel, ss, se, os, oe = input
relation = self.rel_vocab.safe_pad_token
l = len(tokens)
subj_positions = get_positions(ss, se, l)
obj_positions = get_positions(os, oe, l)
subj_type = ner[ss]
obj_type = ner[os]
# anonymize tokens
tokens[ss:se + 1] = ['SUBJ-' + subj_type] * (se - ss + 1)
tokens[os:oe + 1] = ['OBJ-' + obj_type] * (oe - os + 1)
# min head is 0, but root is not included in tokens, so take 1 off from each head
head = [h - 1 for h in head]
yield (tokens, pos, ner, head, deprel, subj_positions,
obj_positions, subj_type, obj_type), relation
def create_types_shapes_values(self) -> Tuple[Tuple, Tuple, Tuple]:
# (tokens, pos, ner, head, deprel, subj_positions, obj_positions, subj_type, obj_type), relation
types = (tf.string, tf.string, tf.string, tf.int32, tf.string,
tf.int32, tf.int32, tf.string, tf.string), tf.string
shapes = ([None], [None], [None], [None], [None], [None], [None], [],
[]), []
pads = (self.token_vocab.safe_pad_token, self.pos_vocab.safe_pad_token,
self.ner_vocab.safe_pad_token, 0,
self.deprel_vocab.safe_pad_token, 0, 0,
self.ner_vocab.safe_pad_token,
self.ner_vocab.safe_pad_token), self.rel_vocab.safe_pad_token
return types, shapes, pads
def x_to_idx(self, x) -> Union[tf.Tensor, Tuple]:
tokens, pos, ner, head, deprel, subj_positions, obj_positions, subj_type, obj_type = x
tokens = self.token_vocab.lookup(tokens)
pos = self.pos_vocab.lookup(pos)
ner = self.ner_vocab.lookup(ner)
deprel = self.deprel_vocab.lookup(deprel)
subj_type = self.ner_vocab.lookup(subj_type)
obj_type = self.ner_vocab.lookup(obj_type)
return tokens, pos, ner, head, deprel, subj_positions, obj_positions, subj_type, obj_type
def y_to_idx(self, y) -> tf.Tensor:
return self.rel_vocab.lookup(y)