def runTests(tests, databases, registerUnicode, iteration=10):
f = ReadingFactory()
timing = {}
for no in tests:
print "Running test %d (reading from %s)..." % (no, databases[no])
connection = {'sqlalchemy.url': 'sqlite:///%s' % databases[no],
'attach': ['cjklib'],
'registerUnicode': registerUnicode[no]}
db = dbconnector.getDBConnector(connection)
availableDicts = [dictClass.DICTIONARY_TABLE for dictClass
in dictionary.BaseDictionary\
.getAvailableDictionaries(db)]
dictionaries = list(set(availableDicts)
& set(db.engine.table_names(schema=db._mainSchema)))
if not dictionaries:
raise ValueError("No dictionaries found")
print "Found dictionaries '%s'" % "', '".join(dictionaries)
runTime = {}
for dictName in dictionaries:
dictClass = dictionary.BaseDictionary.getDictionaryClass(dictName)
dictInstance = dictClass(dbConnectInst=db)
opClass = (dictClass.READING
and f.getReadingOperatorClass(dictClass.READING))
if hasattr(opClass, 'guessReadingDialect'):
requestList = []
for request in SEARCH_REQUESTS:
options = opClass.guessReadingDialect(request)
requestList.append((request, options))
else:
requestList = [(request, {}) for request in SEARCH_REQUESTS]
mod = imp.new_module('timeit_runmod')
mod.runRequest = runRequest
mod.dictInstance = dictInstance
mod.requestList = requestList
sys.modules['timeit_runmod'] = mod
methodTime = {}
for method in ('getFor', 'getForHeadword', 'getForReading',
'getForTranslation'):
t = Timer("""timeit_runmod.runRequest(
timeit_runmod.dictInstance,
timeit_runmod.requestList,
method='%s')
""" % method,
"import timeit_runmod")
methodTime[method] = t.timeit(iteration)
runTime[dictName] = methodTime
timing[no] = runTime
return timing
def open (self, dbname):
"""Open the database."""
self.dbname = dbname
if not hasattr(self, '_dictionaryName'):
self._dictionaryName = dbname
try:
self._dictInst = getDictionary(self._dictionaryName,
entryFactory=entry.UnifiedHeadword())
except ValueError as e:
if debug: print(e, file=sys.stderr)
return False
if self._dictInst.READING:
f = ReadingFactory()
opClass = f.getReadingOperatorClass(self._dictInst.READING)
if hasattr(opClass, 'guessReadingDialect'):
self._opClass = opClass
return True
def runTests(tests, databases, registerUnicode, iteration=10):
f = ReadingFactory()
timing = {}
for no in tests:
print("Running test %d (reading from %s)..." % (no, databases[no]))
connection = {
'sqlalchemy.url': 'sqlite:///%s' % databases[no],
'attach': ['cjklib'],
'registerUnicode': registerUnicode[no]
}
db = dbconnector.getDBConnector(connection)
availableDicts = [dictClass.DICTIONARY_TABLE for dictClass
in dictionary.BaseDictionary\
.getAvailableDictionaries(db)]
dictionaries = list(
set(availableDicts)
& set(db.engine.table_names(schema=db._mainSchema)))
if not dictionaries:
raise ValueError("No dictionaries found")
print("Found dictionaries '%s'" % "', '".join(dictionaries))
runTime = {}
for dictName in dictionaries:
dictClass = dictionary.BaseDictionary.getDictionaryClass(dictName)
dictInstance = dictClass(dbConnectInst=db)
opClass = (dictClass.READING
and f.getReadingOperatorClass(dictClass.READING))
if hasattr(opClass, 'guessReadingDialect'):
requestList = []
for request in SEARCH_REQUESTS:
options = opClass.guessReadingDialect(request)
requestList.append((request, options))
else:
requestList = [(request, {}) for request in SEARCH_REQUESTS]
mod = imp.new_module('timeit_runmod')
mod.runRequest = runRequest
mod.dictInstance = dictInstance
mod.requestList = requestList
sys.modules['timeit_runmod'] = mod
methodTime = {}
for method in ('getFor', 'getForHeadword', 'getForReading',
'getForTranslation'):
t = Timer(
"""timeit_runmod.runRequest(
timeit_runmod.dictInstance,
timeit_runmod.requestList,
method='%s')
""" % method, "import timeit_runmod")
methodTime[method] = t.timeit(iteration)
runTime[dictName] = methodTime
timing[no] = runTime
return timing
class CharacterLookupReadingMethodsTest(CharacterLookupTest,
unittest.TestCase):
"""
Runs consistency checks on the reading methods of the
:class:`~cjklib.characterlookup.CharacterLookup` class.
.. todo::
* Impl: include script table from Unicode 5.2.0 to get character ranges
for Hangul and Kana
"""
DIALECTS = {}
SPECIAL_ENTITY_LIST = {}
def setUp(self):
CharacterLookupTest.setUp(self)
self.f = ReadingFactory(dbConnectInst=self.db)
def testReadingMappingAvailability(self):
"""
Test if the readings under
``CharacterLookup.CHARARACTER_READING_MAPPING`` are available for
conversion.
"""
# mock to simulate availability of all tables in
# characterLookup.CHARARACTER_READING_MAPPING
tables = [table for table, _ \
in list(self.characterLookup.CHARARACTER_READING_MAPPING.values())]
self.characterLookup.db.engine = EngineMock(
self.characterLookup.db.engine, mockTables=tables)
for reading in self.characterLookup.CHARARACTER_READING_MAPPING:
# only if table exists
table, _ = self.characterLookup.CHARARACTER_READING_MAPPING[
reading]
self.assertTrue(
self.characterLookup.hasMappingForReadingToCharacter(reading))
self.assertTrue(
self.characterLookup.hasMappingForCharacterToReading(reading))
# test proper checking for all known readings
for reading in self.f.getSupportedReadings():
self.assertTrue(
self.characterLookup.hasMappingForReadingToCharacter(reading) \
in [True, False])
self.assertTrue(
self.characterLookup.hasMappingForCharacterToReading(reading) \
in [True, False])
@attr('slow')
def testGetCharactersForReadingAcceptsAllEntities(self):
"""Test if ``getCharactersForReading`` accepts all reading entities."""
for reading in self.f.getSupportedReadings():
if not self.characterLookup.hasMappingForReadingToCharacter(
reading):
continue
dialects = [{}]
if reading in self.DIALECTS:
dialects.extend(self.DIALECTS[reading])
for dialect in dialects:
if hasattr(self.f.getReadingOperatorClass(reading),
'getReadingEntities'):
entities = self.f.getReadingEntities(reading, **dialect)
elif reading in self.SPECIAL_ENTITY_LIST:
entities = self.SPECIAL_ENTITY_LIST[reading]
else:
continue
for entity in entities:
try:
results = self.characterLookup.getCharactersForReading(
entity, reading, **dialect)
self.assertEqual(type(results), type([]),
"Method getCharactersForReading() doesn't return" \
+ " a list for entity %s " % repr(entity) \
+ ' (reading %s, dialect %s)' % (reading, dialect))
for entry in results:
self.assertEqual(len(entry), 1,
"Entry %s in result for %s has length != 1" \
% (repr(entry), repr(entity)) \
+ ' (reading %s, dialect %s)' \
% (reading, dialect))
except exception.UnsupportedError:
pass
except exception.ConversionError:
pass
class CharacterLookupReadingMethodsTest(CharacterLookupTest, unittest.TestCase):
"""
Runs consistency checks on the reading methods of the
:class:`~cjklib.characterlookup.CharacterLookup` class.
.. todo::
* Impl: include script table from Unicode 5.2.0 to get character ranges
for Hangul and Kana
"""
DIALECTS = {}
SPECIAL_ENTITY_LIST = {}
def setUp(self):
CharacterLookupTest.setUp(self)
self.f = ReadingFactory(dbConnectInst=self.db)
def testReadingMappingAvailability(self):
"""
Test if the readings under
``CharacterLookup.CHARARACTER_READING_MAPPING`` are available for
conversion.
"""
# mock to simulate availability of all tables in
# characterLookup.CHARARACTER_READING_MAPPING
tables = [table for table, _ \
in self.characterLookup.CHARARACTER_READING_MAPPING.values()]
self.characterLookup.db.engine = EngineMock(
self.characterLookup.db.engine, mockTables=tables)
for reading in self.characterLookup.CHARARACTER_READING_MAPPING:
# only if table exists
table, _ = self.characterLookup.CHARARACTER_READING_MAPPING[reading]
self.assert_(
self.characterLookup.hasMappingForReadingToCharacter(reading))
self.assert_(
self.characterLookup.hasMappingForCharacterToReading(reading))
# test proper checking for all known readings
for reading in self.f.getSupportedReadings():
self.assert_(
self.characterLookup.hasMappingForReadingToCharacter(reading) \
in [True, False])
self.assert_(
self.characterLookup.hasMappingForCharacterToReading(reading) \
in [True, False])
@attr('slow')
def testGetCharactersForReadingAcceptsAllEntities(self):
"""Test if ``getCharactersForReading`` accepts all reading entities."""
for reading in self.f.getSupportedReadings():
if not self.characterLookup.hasMappingForReadingToCharacter(
reading):
continue
dialects = [{}]
if reading in self.DIALECTS:
dialects.extend(self.DIALECTS[reading])
for dialect in dialects:
if hasattr(self.f.getReadingOperatorClass(reading),
'getReadingEntities'):
entities = self.f.getReadingEntities(reading, **dialect)
elif reading in self.SPECIAL_ENTITY_LIST:
entities = self.SPECIAL_ENTITY_LIST[reading]
else:
continue
for entity in entities:
try:
results = self.characterLookup.getCharactersForReading(
entity, reading, **dialect)
self.assertEquals(type(results), type([]),
"Method getCharactersForReading() doesn't return" \
+ " a list for entity %s " % repr(entity) \
+ ' (reading %s, dialect %s)' % (reading, dialect))
for entry in results:
self.assertEquals(len(entry), 1,
"Entry %s in result for %s has length != 1" \
% (repr(entry), repr(entity)) \
+ ' (reading %s, dialect %s)' \
% (reading, dialect))
except exception.UnsupportedError:
pass
except exception.ConversionError:
pass
