def setUp(self):
self.tokens = ['a', 'b', 'c', 'd', 'e', 'f']
corpus = ['a b c', 'b c d d', 'c b e c f']
ids = ['u1', 'u2', 'u3']
# frequency ranking: c > b > d > a > e > f
self.module = TextModule(corpus=corpus, ids=ids, max_vocab=6)
self.module.build({'u1': 0, 'u2': 1, 'u3': 2})
self.token_ids = (self.module.vocab.tok2idx[tok]
for tok in self.tokens)
def test_with_modules():
from cornac.data import TextModule, ImageModule, GraphModule
bm = BaseMethod()
assert bm.user_text is None
assert bm.item_text is None
assert bm.user_image is None
assert bm.item_image is None
assert bm.user_graph is None
assert bm.item_graph is None
bm.user_text = TextModule()
#bm.item_graph = GraphModule()
bm._build_modules()
try:
bm.user_text = ImageModule()
except ValueError:
assert True
#try:
# bm.item_text = GraphModule()
#except ValueError:
# assert True
try:
bm.user_image = TextModule()
except ValueError:
assert True
#try:
# bm.item_image = GraphModule()
#except ValueError:
# assert True
try:
bm.user_graph = TextModule()
except ValueError:
assert True
try:
bm.item_graph = ImageModule()
except ValueError:
assert True
def test_with_modules(self):
bm = BaseMethod()
self.assertIsNone(bm.user_text)
self.assertIsNone(bm.item_text)
self.assertIsNone(bm.user_image)
self.assertIsNone(bm.item_image)
self.assertIsNone(bm.user_graph)
self.assertIsNone(bm.item_graph)
bm.user_text = TextModule()
bm.item_image = ImageModule()
bm._build_modules()
try:
bm.user_text = ImageModule()
except ValueError:
assert True
try:
bm.item_text = ImageModule()
except ValueError:
assert True
try:
bm.user_image = TextModule()
except ValueError:
assert True
try:
bm.item_image = TextModule()
except ValueError:
assert True
try:
bm.user_graph = TextModule()
except ValueError:
assert True
try:
bm.item_graph = ImageModule()
except ValueError:
assert True
"""
import cornac
from cornac.data import Reader
from cornac.datasets import citeulike
from cornac.eval_methods import RatioSplit
from cornac.data import TextModule
from cornac.data.text import BaseTokenizer
docs, item_ids = citeulike.load_text()
data = citeulike.load_data(reader=Reader(item_set=item_ids))
# build text module
item_text_module = TextModule(corpus=docs,
ids=item_ids,
tokenizer=BaseTokenizer('\t'),
max_vocab=8000,
max_doc_freq=0.5,
stop_words='english')
ratio_split = RatioSplit(data=data,
test_size=0.2,
exclude_unknowns=True,
item_text=item_text_module,
verbose=True,
seed=123,
rating_threshold=0.5)
cdr = cornac.models.CDR(k=50,
autoencoder_structure=[200],
max_iter=100,
batch_size=128,
@author: Tran Thanh Binh
"""
import cornac
from cornac.data import Reader
from cornac.datasets import citeulike
from cornac.eval_methods import RatioSplit
from cornac.data import TextModule
docs, item_ids = citeulike.load_text()
data = citeulike.load_data(reader=Reader(item_set=item_ids))
# build text module
item_text_module = TextModule(corpus=docs,
ids=item_ids,
max_vocab=8000,
max_doc_freq=0.5,
stop_words='english')
ratio_split = RatioSplit(data=data,
test_size=0.2,
exclude_unknowns=True,
item_text=item_text_module,
verbose=True,
seed=123,
rating_threshold=0.5)
cvae = cornac.models.CVAE(n_epochs=20, seed=123, verbose=True)
rec_300 = cornac.metrics.Recall(k=300)
"""Example for Convolutional Matrix Factorization"""
import cornac
from cornac.data import Reader
from cornac.datasets import movielens
from cornac.eval_methods import RatioSplit
from cornac.data import TextModule
from cornac.data.text import BaseTokenizer
plots, movie_ids = movielens.load_plot()
ml_1m = movielens.load_1m(reader=Reader(item_set=movie_ids))
# build text module
item_text_module = TextModule(corpus=plots,
ids=movie_ids,
tokenizer=BaseTokenizer(sep='\t',
stop_words='english'),
max_vocab=8000,
max_doc_freq=0.5)
ratio_split = RatioSplit(data=ml_1m,
test_size=0.2,
exclude_unknowns=True,
item_text=item_text_module,
verbose=True,
seed=123)
convmf = cornac.models.ConvMF(n_epochs=5, verbose=True, seed=123)
rmse = cornac.metrics.RMSE()
exp = cornac.Experiment(eval_method=ratio_split,
def test_batch_bow_fallback(self):
module = TextModule(features=np.asarray([[3, 2, 1], [4, 5, 6]]),
ids=['a', 'b'])
module.build()
npt.assert_array_equal(np.asarray([[3, 2, 1]]),
module.batch_bow(batch_ids=[0]))
def test_build(self):
TextModule().build()
TextModule(corpus=['abc']).build()
TextModule(corpus=['abc']).build({'b': 0})
TextModule(corpus=['abc'], ids=['a']).build({'b': 0})
class TestTextModule(unittest.TestCase):
def setUp(self):
self.tokens = ['a', 'b', 'c', 'd', 'e', 'f']
corpus = ['a b c', 'b c d d', 'c b e c f']
ids = ['u1', 'u2', 'u3']
# frequency ranking: c > b > d > a > e > f
self.module = TextModule(corpus=corpus, ids=ids, max_vocab=6)
self.module.build({'u1': 0, 'u2': 1, 'u3': 2})
self.token_ids = (self.module.vocab.tok2idx[tok]
for tok in self.tokens)
def test_init(self):
self.assertCountEqual(self.module.vocab.idx2tok,
SPECIAL_TOKENS + self.tokens)
def test_build(self):
TextModule().build()
TextModule(corpus=['abc']).build()
TextModule(corpus=['abc']).build({'b': 0})
TextModule(corpus=['abc'], ids=['a']).build({'b': 0})
def test_sequences(self):
(a, b, c, d, e, f) = self.token_ids
self.assertListEqual(self.module.sequences,
[[a, b, c], [b, c, d, d], [c, b, e, c, f]])
def test_batch_seq(self):
(a, b, c, d, e, f) = self.token_ids
batch_seqs = self.module.batch_seq([2, 1])
self.assertEqual((2, 5), batch_seqs.shape)
npt.assert_array_equal(batch_seqs,
np.asarray([[c, b, e, c, f], [b, c, d, d, 0]]))
batch_seqs = self.module.batch_seq([0, 2], max_length=4)
self.assertEqual((2, 4), batch_seqs.shape)
npt.assert_array_equal(batch_seqs,
np.asarray([[a, b, c, 0], [c, b, e, c]]))
self.module.sequences = None
try:
self.module.batch_seq([0])
except ValueError:
assert True
def test_count_matrix(self):
(a, b, c, d, e, f) = self.token_ids
shift = len(SPECIAL_TOKENS)
expected_counts = np.zeros_like(self.module.count_matrix.A)
expected_counts[0, a - shift] = 1
expected_counts[0, b - shift] = 1
expected_counts[0, c - shift] = 1
expected_counts[1, b - shift] = 1
expected_counts[1, c - shift] = 1
expected_counts[1, d - shift] = 2
expected_counts[2, b - shift] = 1
expected_counts[2, c - shift] = 2
expected_counts[2, e - shift] = 1
expected_counts[2, f - shift] = 1
npt.assert_array_equal(self.module.count_matrix.A, expected_counts)
def test_batch_bow(self):
(a, b, c, d, e, f) = self.token_ids
shift = len(SPECIAL_TOKENS)
batch_bows = self.module.batch_bow([2, 1])
self.assertEqual((2, self.module.max_vocab), batch_bows.shape)
expected_bows = np.zeros_like(batch_bows)
expected_bows[0, b - shift] = 1
expected_bows[0, c - shift] = 2
expected_bows[0, e - shift] = 1
expected_bows[0, f - shift] = 1
expected_bows[1, b - shift] = 1
expected_bows[1, c - shift] = 1
expected_bows[1, d - shift] = 2
npt.assert_array_equal(batch_bows, expected_bows)
batch_bows = self.module.batch_bow([0, 2],
binary=True,
keep_sparse=True)
self.assertEqual((2, 6), batch_bows.shape)
expected_bows = np.zeros_like(batch_bows.A)
expected_bows[0, np.asarray([a, b, c]) - shift] = 1
expected_bows[1, np.asarray([b, c, e, f]) - shift] = 1
npt.assert_array_equal(batch_bows.A, expected_bows)
self.module.count_matrix = None
try:
self.module.batch_bow([0])
except ValueError:
assert True
def test_batch_bow_fallback(self):
module = TextModule(features=np.asarray([[3, 2, 1], [4, 5, 6]]),
ids=['a', 'b'])
module.build()
npt.assert_array_equal(np.asarray([[3, 2, 1]]),
module.batch_bow(batch_ids=[0]))
def test_init():
md = TextModule()
md.build(global_id_map=None)
