def __init__(self, *args, **kwargs):
super(CacheTest, self).__init__(*args, **kwargs)
corparch = None
mocked_json = [{
'ident': 'wiktionary_for_ic_9_en',
'heading': {'en_US': 'Wiktionary', 'cs_CZ': 'Wiktionary'},
'backend': 'plugins.default_token_connect.mock_http_backend.MockHTTPBackend',
'frontend': 'plugins.default_token_connect.frontends.RawHtmlFrontend',
'conf':
{
'server': 'en.wiktionary.org',
'path': '/w/index.php?title={lemma}&action=render',
'ssl': True,
'port': 443,
'posAttrs': ['lemma']
}
}]
providers_conf = mocked_json
self.tok_det = DefaultTokenConnect(dict((b['ident'], init_provider(b, b['ident'])) for b in providers_conf),
corparch)
self.cache_path = CACHE_DB_PATH
cache_rows_limit = 10
cache_ttl_days = 7
self.cache_man = CacheMan(self.cache_path, cache_rows_limit, cache_ttl_days)
self.tok_det.set_cache_path(self.cache_path)
def __init__(self, *args, **kwargs):
super(CacheTest, self).__init__(*args, **kwargs)
corparch = None
mocked_json = [{
"ident": "wiktionary_for_ic_9_en",
"heading": {
"en_US": "Wiktionary",
"cs_CZ": "Wiktionary"
},
"backend":
"plugins.default_token_connect.mock_http_backend.HTTPBackend",
"frontend":
"plugins.default_token_connect.frontends.RawHtmlFrontend",
"conf": {
"server": "en.wiktionary.org",
"path": "/w/index.php?title={lemma}&action=render",
"ssl": True,
"port": 443
}
}]
providers_conf = mocked_json
self.tok_det = DefaultTokenConnect(
dict((b['ident'], init_provider(b, b['ident']))
for b in providers_conf), corparch)
self.cache_path = '/tmp/token_cache/token_connect_cache.db'
cache_rows_limit = 10
cache_ttl_days = 7
self.cacheMan = CacheMan(self.cache_path, cache_rows_limit,
cache_ttl_days)
self.tok_det.set_cache_path(self.cache_path)
def setup_providers(plg_conf):
with open(plg_conf['default:providers_conf'], 'rb') as fr:
providers_conf = json.load(fr)
cache_path = plg_conf.get('default:cache_db_path')
providers = dict((b['ident'], init_provider(b, b['ident'])) for b in providers_conf)
if cache_path and not os.path.isfile(cache_path):
cache_manager = CacheMan(cache_path)
cache_manager.prepare_cache()
return providers, cache_path
def clean_cache(cache_size):
CacheMan(self.cache_path).connect().clear_extra_rows(cache_size)
class CacheTest(unittest.TestCase):
def __init__(self, *args, **kwargs):
super(CacheTest, self).__init__(*args, **kwargs)
corparch = None
mocked_json = [{
'ident': 'wiktionary_for_ic_9_en',
'heading': {'en_US': 'Wiktionary', 'cs_CZ': 'Wiktionary'},
'backend': 'plugins.default_token_connect.mock_http_backend.MockHTTPBackend',
'frontend': 'plugins.default_token_connect.frontends.RawHtmlFrontend',
'conf':
{
'server': 'en.wiktionary.org',
'path': '/w/index.php?title={lemma}&action=render',
'ssl': True,
'port': 443,
'posAttrs': ['lemma']
}
}]
providers_conf = mocked_json
self.tok_det = DefaultTokenConnect(dict((b['ident'], init_provider(b, b['ident'])) for b in providers_conf),
corparch)
self.cache_path = CACHE_DB_PATH
cache_rows_limit = 10
cache_ttl_days = 7
self.cache_man = CacheMan(self.cache_path, cache_rows_limit, cache_ttl_days)
self.tok_det.set_cache_path(self.cache_path)
def setUp(self):
"""
create an empty cache db file with properly structured table
"""
self.cache_man.prepare_cache()
def tearDown(self):
self.cache_man.close()
def raise_exc(self):
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], "corpora",
"lang", dict(lemma=u"exception"), 1, 1)
def test_get_path(self):
"""
check whether the cache path set above using set_cache_path is returned using get_cache_path
"""
self.assertEqual(self.cache_man.get_cache_path(), self.cache_path)
def test_cache_item(self):
"""
fetch two items from http backend, check whether they get stored in cache by checking number of rows
"""
mc = MockCorpus({1: u'lemma1', 2: u'lemma2'})
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
self.assertEqual(self.cache_man.get_numrows(), 2)
def test_retrieve_cached_item(self):
"""
fetch two items from http backend, they should be cached
then fetch the same two items again and check whether they are retrieved correctly from cache
and that the cache contains only two items
"""
mc = MockCorpus({1: u'lemma1', 2: u'lemma2'})
orig1 = self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
orig2 = self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
cached1 = self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
cached2 = self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
self.assertEqual(self.cache_man.get_numrows(), 2)
self.assertEqual(orig1, cached1)
self.assertEqual(orig2, cached2)
def test_last_access(self):
"""
fetch two items from http backend to cache them, get their last access value from cache
then fetch the same two items again after a time interval and check whether the last access values changed
"""
mc = MockCorpus({1: u'lemma1', 2: u'lemma2'})
key1 = mk_token_connect_cache_key("wiktionary_for_ic_9_en", [
"corpora"], "lang", dict(lemma=u'lemma1'))
key2 = mk_token_connect_cache_key("wiktionary_for_ic_9_en", [
"corpora"], "lang", dict(lemma=u'lemma2'))
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
la1bef = self.get_last_access(key1)
la2bef = self.get_last_access(key2)
time.sleep(1)
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
time.sleep(1)
self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
la1aft = self.get_last_access(key1)
la2aft = self.get_last_access(key2)
self.assertNotEqual(la1bef, la1aft)
self.assertNotEqual(la2bef, la2aft)
def test_clear_cache(self):
"""
fill the cache db with excessive number of rows, run cache maintenance, check whether size was decreased
to the limit
"""
limit = self.cache_man.get_rows_limit()
self.fill_cache(limit + 10)
self.cache_man.clear_extra_rows()
self.assertEqual(self.cache_man.get_numrows(), limit)
def test_unicode(self):
"""
cache a unicode encoded string, check whether it gets returned correctly
the unicode-encoded result is returned from the mocked backend when searching for lemma "unicode"
"""
mc = MockCorpus({1: u'unicode'})
orig1 = self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
cached1 = self.tok_det.fetch_data(['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
self.assertEqual(orig1, cached1)
def test_status_true_false(self):
"""
cache a "found" and a "not-found" result returned by the mocked backend, check whether
the "found" / "not-found" information gets cached and retrieved correctly
"""
mc = MockCorpus()
_, orig1 = self.tok_det.fetch_data(
['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
_, cached1 = self.tok_det.fetch_data(
['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 1, 1)
_, orig2 = self.tok_det.fetch_data(
['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
_, cached2 = self.tok_det.fetch_data(
['wiktionary_for_ic_9_en'], mc, ["corpora"], "lang", 2, 1)
self.assertEqual(orig1[0].get('contents')[0][1][1],
cached1[0].get('contents')[0][1][1], False)
self.assertEqual(orig2[0].get('contents')[0][1][1],
cached2[0].get('contents')[0][1][1], True)
def test_backend_exception(self):
self.assertRaises(Exception, self.raise_exc)
# -----------
# aux methods
# -----------
def list_cached(self):
print "--- cache contents: ---"
conn = self.cache_man.conn
c = conn.cursor()
for row in c.execute("SELECT * FROM cache"):
print row
print "------"
def fill_cache(self, numrows=10):
conn = self.cache_man.conn
c = conn.cursor()
for i in range(0, numrows):
c.execute("INSERT INTO cache VALUES (?, ?, ?, ?, ?)", (i, 'some-provider', i, True, i))
conn.commit()
def get_specific_row(self, key):
conn = self.cache_man.conn
c = conn.cursor()
res = c.execute("SELECT data FROM cache WHERE key = ?", (key,)).fetchone()
return res
def get_last_access(self, key):
conn = self.cache_man.conn
c = conn.cursor()
last_access = c.execute("SELECT last_access FROM cache WHERE key = ?", (key,)).fetchone()
if last_access:
return last_access[0]
else:
return 0
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
import os
import sys
sys.path.insert(
0,
os.path.realpath(os.path.join(os.path.dirname(__file__),
'../../../../lib')))
import settings
from plugins.default_token_connect.cache_man import CacheMan
conf_path = os.path.realpath(
os.path.join(os.path.dirname(__file__), '../../../../conf/config.xml'))
settings.load(conf_path)
conf = settings.get('plugins', 'token_connect')
cache_path = conf.get('default:cache_db_path')
if cache_path:
cache_rows_limit = conf.get('default:cache_rows_limit')
cache_ttl_days = conf.get('default:cache_ttl_days')
cacheMan = CacheMan(cache_path, cache_rows_limit, cache_ttl_days)
cacheMan.clear_extra_rows()