def test_id2doc(self):
# word ids.
docs = [['B-PSN'], ['B-ORG', 'I-ORG'], ['B-LOC', 'I-LOC', 'O']]
vocab = Vocabulary(unk_token=False, lower=False)
vocab.add_documents(docs)
vocab.build()
true_doc = ['O', 'B-LOC', 'O', 'O']
doc_ids = vocab.doc2id(true_doc)
pred_doc = vocab.id2doc(doc_ids)
self.assertEqual(pred_doc, true_doc)
def fit(self, X, y, y_class = None):
"""Learn vocabulary from training set.
Args:
X : iterable. An iterable which yields either str, unicode or file objects.
Returns:
self : IndexTransformer.
"""
self._word_vocab.add_documents(X)
self._label_vocab.add_documents(y)
if self._use_char:
for doc in X:
self._char_vocab.add_documents(doc)
if y_class is not None:
for cat in y_class:
cat_vocab = Vocabulary(lower=False, unk_token='UNK')
cat_vocab.add_documents(cat)
cat_vocab.build()
print(cat_vocab.vocab)
self._label_class_vocab.append(cat_vocab)
self._word_vocab.build()
self._char_vocab.build()
self._label_vocab.build()
return self
def __init__(self,
lower=True,
num_norm=True,
use_char=True,
initial_vocab=None,
max_size=None,
min_freq=1,
alphanumerical=False,
max_sent_len=100,
max_word_len=20):
"""Create a preprocessor object.
Args:
lower: boolean. Whether to convert the texts to lowercase.
use_char: boolean. Whether to use char feature.
num_norm: boolean. Whether to normalize text.
initial_vocab: Iterable. Initial vocabulary for expanding word_vocab.
"""
self._num_norm = num_norm
self._use_char = use_char
self._word_vocab = Vocabulary(lower=lower,
max_size=max_size,
min_freq=min_freq,
alphanumerical=alphanumerical)
self._char_vocab = Vocabulary(lower=False)
self._label_vocab = Vocabulary(lower=False, unk_token=False)
# currently not implemented
self._max_sent_len = max_sent_len
self._max_word_len = max_word_len
if initial_vocab:
self._word_vocab.add_documents([initial_vocab])
self._char_vocab.add_documents(initial_vocab)
def __init__(self, lower=True, num_norm=True,
use_char=True, initial_vocab=None):
"""Create a preprocessor object.
Args:
lower: boolean. Whether to convert the texts to lowercase.
use_char: boolean. Whether to use char feature.
num_norm: boolean. Whether to normalize text.
initial_vocab: Iterable. Initial vocabulary for expanding word_vocab.
"""
self._num_norm = num_norm
self._use_char = use_char
self._word_vocab = Vocabulary(lower=lower)
self._char_vocab = Vocabulary(lower=False)
self._label_vocab = Vocabulary(lower=False, unk_token=False)
if initial_vocab:
self._word_vocab.add_documents([initial_vocab])
self._char_vocab.add_documents(initial_vocab)
class IndexTransformer(BaseEstimator, TransformerMixin):
"""Convert a collection of raw documents to a document id matrix.
Attributes:
_use_char: boolean. Whether to use char feature.
_num_norm: boolean. Whether to normalize text.
_word_vocab: dict. A mapping of words to feature indices.
_char_vocab: dict. A mapping of chars to feature indices.
_label_vocab: dict. A mapping of labels to feature indices.
"""
def __init__(self, lower=True, num_norm=True,
use_char=True, initial_vocab=None):
"""Create a preprocessor object.
Args:
lower: boolean. Whether to convert the texts to lowercase.
use_char: boolean. Whether to use char feature.
num_norm: boolean. Whether to normalize text.
initial_vocab: Iterable. Initial vocabulary for expanding word_vocab.
"""
self._num_norm = num_norm
self._use_char = use_char
self._word_vocab = Vocabulary(lower=lower)
self._char_vocab = Vocabulary(lower=False)
self._label_vocab = Vocabulary(lower=False, unk_token=False)
if initial_vocab:
self._word_vocab.add_documents([initial_vocab])
self._char_vocab.add_documents(initial_vocab)
def fit(self, X, y):
"""Learn vocabulary from training set.
Args:
X : iterable. An iterable which yields either str, unicode or file objects.
Returns:
self : IndexTransformer.
"""
self._word_vocab.add_documents(X)
self._label_vocab.add_documents(y)
if self._use_char:
for doc in X:
self._char_vocab.add_documents(doc)
self._word_vocab.build()
self._char_vocab.build()
self._label_vocab.build()
return self
def transform(self, X, y=None):
"""Transform documents to document ids.
Uses the vocabulary learned by fit.
Args:
X : iterable
an iterable which yields either str, unicode or file objects.
y : iterabl, label strings.
Returns:
features: document id matrix.
y: label id matrix.
"""
word_ids = [self._word_vocab.doc2id(doc) for doc in X]
word_ids = pad_sequences(word_ids, padding='post')
if self._use_char:
char_ids = [[self._char_vocab.doc2id(w) for w in doc] for doc in X]
char_ids = pad_nested_sequences(char_ids)
features = [word_ids, char_ids]
else:
features = word_ids
if y is not None:
y = [self._label_vocab.doc2id(doc) for doc in y]
y = pad_sequences(y, padding='post')
y = to_categorical(y, self.label_size).astype(int)
# In 2018/06/01, to_categorical is a bit strange.
# >>> to_categorical([[1,3]], num_classes=4).shape
# (1, 2, 4)
# >>> to_categorical([[1]], num_classes=4).shape
# (1, 4)
# So, I expand dimensions when len(y.shape) == 2.
y = y if len(y.shape) == 3 else np.expand_dims(y, axis=0)
return features, y
else:
return features
def fit_transform(self, X, y=None, **params):
"""Learn vocabulary and return document id matrix.
This is equivalent to fit followed by transform.
Args:
X : iterable
an iterable which yields either str, unicode or file objects.
Returns:
list : document id matrix.
list: label id matrix.
"""
return self.fit(X, y).transform(X, y)
def inverse_transform(self, y, lengths=None):
"""Return label strings.
Args:
y: label id matrix.
lengths: sentences length.
Returns:
list: list of list of strings.
"""
y = np.argmax(y, -1)
inverse_y = [self._label_vocab.id2doc(ids) for ids in y]
if lengths is not None:
inverse_y = [iy[:l] for iy, l in zip(inverse_y, lengths)]
return inverse_y
@property
def word_vocab_size(self):
return len(self._word_vocab)
@property
def char_vocab_size(self):
return len(self._char_vocab)
@property
def label_size(self):
return len(self._label_vocab)
def save(self, file_path):
joblib.dump(self, file_path)
@classmethod
def load(cls, file_path):
p = joblib.load(file_path)
return p
def test_doc2id(self):
# word ids.
docs = [['a'], ['a', 'b'], ['a', 'b', 'c']]
vocab = Vocabulary()
vocab.add_documents(docs)
vocab.build()
another_doc = ['a', 'b', 'c', 'd']
doc_ids = vocab.doc2id(another_doc)
self.assertEqual(doc_ids, [1, 2, 3, 4])
# char_ids.
docs = ['hoge', 'fuga', 'bar']
vocab = Vocabulary()
vocab.add_documents(docs)
vocab.build()
doc_ids = vocab.doc2id(docs[0])
correct = [vocab.token_to_id(c) for c in docs[0]]
self.assertEqual(doc_ids, correct)
def test_add_documents(self):
# word vocabulary.
docs = [['a'], ['a', 'b'], ['a', 'b', 'c']]
token2id = {'<pad>': 0, 'a': 1, 'b': 2, 'c': 3, '<unk>': 4}
vocab = Vocabulary()
vocab.add_documents(docs)
vocab.build()
self.assertEqual(vocab._token2id, token2id)
token2id = {'<pad>': 0, 'a': 1, 'b': 2, 'c': 3}
vocab = Vocabulary(unk_token=False)
vocab.add_documents(docs)
vocab.build()
self.assertEqual(vocab._token2id, token2id)
token2id = {'<pad>': 0, '<s>': 1, 'a': 2, 'b': 3, 'c': 4}
vocab = Vocabulary(unk_token=False, specials=('<pad>', '<s>'))
vocab.add_documents(docs)
vocab.build()
self.assertEqual(vocab._token2id, token2id)
token2id = {'a': 0, 'b': 1, 'c': 2}
vocab = Vocabulary(unk_token=False, specials=())
vocab.add_documents(docs)
vocab.build()
self.assertEqual(vocab._token2id, token2id)
# char vocabulary.
docs = ['hoge', 'fuga', 'bar']
vocab = Vocabulary()
vocab.add_documents(docs)
vocab.build()
num_chars = len(set(''.join(docs))) + 2
self.assertEqual(len(vocab._token2id), num_chars)