def process_example(self, example):
"""Create a training example using the given tokens."""
log.debug(example)
example = text_to_vocab_indices(self.vocab, example)[0]
log.debug(example)
return (example[:self.left_context] if self.right_context == 0 else
example[:self.left_context] + example[self.left_context + 1:],
example[self.left_context])
def process_example(self, example):
"""Create a training example using the given tokens."""
log.debug(example)
example = text_to_vocab_indices(self.vocab, example)[0]
log.debug(example)
return (example[:self.left_context] if self.right_context == 0 else
example[:self.left_context] + example[self.left_context + 1:],
example[self.left_context])
def _process_example_full_text(self, example):
"""Process the given example that contains context and target word.
The implementation is taken from SimpleVLblNceTrainer.process_example.
"""
idx, example = text_to_vocab_indices(self.vocab, example)
return (idx[:self.model.left_context] if self.model.right_context == 0 else
idx[:self.model.left_context] + idx[self.model.left_context + 1:],
idx[self.model.left_context]), example
def process_example(self, example):
"""Create a fake example out of the given correct one.
Additionally, replaces invalid token ids by the unknown word.
"""
example = text_to_vocab_indices(self.effective_vocab_size, example)[0]
fake_example = [x for x in example]
fake_example[self.left_context] = randint(0, self.effective_vocab_size - 1)
return (example, fake_example)
def process_example(self, example):
"""Create a fake example out of the given correct one.
Additionally, replaces invalid token ids by the unknown word.
"""
example = text_to_vocab_indices(self.effective_vocab_size, example)[0]
fake_example = [x for x in example]
fake_example[self.left_context] = randint(
0, self.effective_vocab_size - 1)
return (example, fake_example)
def process_example(self, example):
"""Convert the given example in handable data structures.
Splits vectors in their single values and converts the labels into ints
and the data into floats.
Returns
-------
list(str)
input text
"""
# return example.split(' ')
return text_to_vocab_indices(self.vocab, example)[0]
def process_example(self, example):
"""Convert the given example in handable data structures.
Splits vectors in their single values and converts the labels into ints
and the data into floats.
Returns
-------
list(str)
input text
"""
# return example.split(' ')
return text_to_vocab_indices(self.vocab, example)[0]
def _process_example_context_per_line(self, example):
"""Process the given example that contains only the context and not the
target word.
"""
return text_to_vocab_indices(self.vocab, example)
def process_example(self, example):
example = text_to_vocab_indices(self.effective_vocab_size, example)[0]
fake_example = [x for x in example]
fake_example[self.left_context] = self.choose_random_sentiment_word()
return (example, fake_example)
def _process_example_context_per_line(self, example):
"""Process the given example that contains only the context and not the
target word.
"""
return text_to_vocab_indices(self.vocab, example)
def process_example(self, example):
example = text_to_vocab_indices(self.effective_vocab_size, example)[0]
fake_example = [x for x in example]
fake_example[self.left_context] = self.choose_random_sentiment_word()
return (example, fake_example)