def get_new_connection_to_library_database(self):
library_database = LibraryDatabase(self._database_name)
if self._database_name == ':memory:':
# In memory database does not point to the right place..
# this is here for unit tests..
library_database.create_database()
return library_database
def get_new_connection_to_library_database(self):
library_database = LibraryDatabase(self._database_name)
if self._database_name == ':memory:':
# In memory database does not point to the right place..
# this is here for unit tests..
library_database.create_database()
return library_database
def _initiate_database_connection(self):
self._database = LibraryDatabase(self._database_name)
class LibraryManager(Thread):
def __init__(self, database_name, spec_initializer=None):
self._database_name = database_name
self._database = None
self._messages = Queue.Queue()
self._spec_initializer = spec_initializer or SpecInitializer()
Thread.__init__(self)
self.setDaemon(True)
def run(self):
self._initiate_database_connection()
while True:
try:
if not self._handle_message():
break
except Exception as err:
msg = 'Library import handling threw an unexpected exception'
RideLogException(message=msg, exception=err,
level='WARN').publish()
self._database.close()
def _initiate_database_connection(self):
self._database = LibraryDatabase(self._database_name)
def get_new_connection_to_library_database(self):
library_database = LibraryDatabase(self._database_name)
if self._database_name == ':memory:':
# In memory database does not point to the right place..
# this is here for unit tests..
library_database.create_database()
return library_database
def _handle_message(self):
message = self._messages.get()
if not message:
return False
msg_type = message[0]
if msg_type == 'fetch':
self._handle_fetch_keywords_message(message)
elif msg_type == 'insert':
self._handle_insert_keywords_message(message)
elif msg_type == 'create':
self._database.create_database()
return True
def _handle_fetch_keywords_message(self, message):
_, library_name, library_args, callback = message
keywords = self._fetch_keywords(library_name, library_args)
self._update_database_and_call_callback_if_needed(
(library_name, library_args), keywords, callback)
def _fetch_keywords(self, library_name, library_args):
try:
path = get_path(library_name.replace('/', os.sep),
os.path.abspath('.'))
return get_import_result(path, library_args)
except Exception as err:
try:
print('FAILED', library_name, err)
except IOError:
pass
kws = self._spec_initializer.init_from_spec(library_name)
if not kws:
msg = 'Importing test library "%s" failed' % library_name
RideLogException(message=msg, exception=err,
level='WARN').publish()
return kws
def _handle_insert_keywords_message(self, message):
_, library_name, library_args, result_queue = message
keywords = self._fetch_keywords(library_name, library_args)
self._insert(library_name, library_args, keywords,
lambda res: result_queue.put(res, timeout=3))
def _insert(self, library_name, library_args, keywords, callback):
self._database.insert_library_keywords(library_name, library_args,
keywords or [])
self._call(callback, keywords)
def _update_database_and_call_callback_if_needed(self, library_key,
keywords, callback):
db_keywords = self._database.fetch_library_keywords(*library_key)
try:
if not db_keywords or self._keywords_differ(keywords, db_keywords):
self._insert(library_key[0], library_key[1], keywords,
callback)
else:
self._database.update_library_timestamp(*library_key)
except OperationalError:
pass
def _call(self, callback, *args):
try:
callback(*args)
except Exception as err:
msg = 'Library import callback threw an unexpected exception'
RideLogException(message=msg, exception=err,
level='WARN').publish()
def fetch_keywords(self, library_name, library_args, callback):
self._messages.put(('fetch', library_name, library_args, callback),
timeout=3)
def get_and_insert_keywords(self, library_name, library_args):
result_queue = Queue.Queue(maxsize=1)
self._messages.put(
('insert', library_name, library_args, result_queue), timeout=3)
try:
return result_queue.get(timeout=5)
except Queue.Empty as e:
RideLogMessage(
u'Failed to read keywords from library db: {}'.format(
unicode(e))).publish()
return []
def create_database(self):
self._messages.put(('create', ), timeout=3)
def stop(self):
self._messages.put(False, timeout=3)
def _keywords_differ(self, keywords1, keywords2):
if keywords1 != keywords2 and None in (keywords1, keywords2):
return True
if len(keywords1) != len(keywords2):
return True
for k1, k2 in zip(keywords1, keywords2):
if k1.name != k2.name:
return True
if k1.doc != k2.doc:
return True
# if k1.doc_format != k2.doc_format:
# return True
if k1.arguments != k2.arguments:
return True
if k1.source != k2.source:
return True
return False
def setUp(self):
self._database = LibraryDatabase(':memory:')
self._database.create_database()
class TestLibraryDatabase(unittest.TestCase):
def setUp(self):
self._database = LibraryDatabase(':memory:')
self._database.create_database()
def tearDown(self):
self._database.close()
def test_inserting_and_fetching(self):
collections_kws = self._get_and_insert_keywords('Collections', '')
string_kws = self._get_and_insert_keywords('String', '')
builtin_kws = self._get_and_insert_keywords('BuiltIn', '')
collections_kws_from_db = self._database.fetch_library_keywords(
'Collections', '')
string_kws_from_db = self._database.fetch_library_keywords(
'String', '')
builtin_kws_from_db = self._database.fetch_library_keywords(
'BuiltIn', '')
for originals, from_database in [[
collections_kws, collections_kws_from_db
], [string_kws, string_kws_from_db],
[builtin_kws, builtin_kws_from_db]]:
self._check_keywords(originals, from_database)
def test_finds_newest_version(self):
self._database.insert_library_keywords(
'lib.py', 'foo',
[LibraryKeywordInfo('this is old', 'doc', 'lib.py', '')])
self._database.insert_library_keywords(
'lib.py', 'foo',
[LibraryKeywordInfo('this is new', 'doc', 'lib.py', '')])
kws = self._database.fetch_library_keywords('lib.py', 'foo')
self.assertEqual(len(kws), 1, unicode(kws))
self.assertEqual(kws[0].name, 'this is new')
def test_removing_old_data(self):
self._get_and_insert_keywords('String', '')
self._get_and_insert_keywords('String', '')
newest = self._get_and_insert_keywords('String', '')
newest_again = self._database.fetch_library_keywords('String', '')
self._check_keywords(newest, newest_again)
self.assertEqual(
self._database._connection.execute(
'select count(*) from libraries').fetchone()[0], 1)
self.assertEqual(
self._database._connection.execute(
'select count(*) from keywords').fetchone()[0], len(newest))
def test_unknown_library_keywords_fetch(self):
results = self._database.fetch_library_keywords(
'Something that is not there', 'at all')
self.assertEqual(len(results), 0)
def test_unknown_library_fetch(self):
self.assertFalse(self._database.library_exists('library', ''))
self._database.insert_library_keywords('library', '', [])
self.assertTrue(self._database.library_exists('library', ''))
def _get_and_insert_keywords(self, library_name, library_arguments):
kws = get_import_result(library_name, library_arguments)
self._database.insert_library_keywords(library_name, library_arguments,
kws)
return kws
def _check_keywords(self, originals, from_database):
for k1, k2 in zip(originals, from_database):
self.assertEqual(k1.name, k2.name)
self.assertEqual(k1.doc, k2.doc)
self.assertEqual(
k1.arguments, k2.arguments,
'Arguments differ ("%s" != "%s") on keyword %s' %
(k1.arguments, k2.arguments, k1.name))
self.assertEqual(k1.source, k2.source)
self.assertEqual(len(originals), len(from_database))
def setUp(self):
self._database = LibraryDatabase(':memory:')
self._database.create_database()
class TestLibraryDatabase(unittest.TestCase):
def setUp(self):
self._database = LibraryDatabase(':memory:')
self._database.create_database()
def tearDown(self):
self._database.close()
def test_inserting_and_fetching(self):
collections_kws = self._get_and_insert_keywords('Collections', '')
string_kws = self._get_and_insert_keywords('String', '')
builtin_kws = self._get_and_insert_keywords('BuiltIn', '')
collections_kws_from_db = self._database.fetch_library_keywords('Collections', '')
string_kws_from_db = self._database.fetch_library_keywords('String', '')
builtin_kws_from_db = self._database.fetch_library_keywords('BuiltIn', '')
for originals, from_database in [[collections_kws, collections_kws_from_db],
[string_kws, string_kws_from_db],
[builtin_kws, builtin_kws_from_db]]:
self._check_keywords(originals, from_database)
def test_finds_newest_version(self):
self._database.insert_library_keywords('lib.py', 'foo', [LibraryKeywordInfo('this is old', 'doc', 'lib.py', '')])
self._database.insert_library_keywords('lib.py', 'foo', [LibraryKeywordInfo('this is new', 'doc', 'lib.py', '')])
kws = self._database.fetch_library_keywords('lib.py', 'foo')
self.assertEqual(len(kws), 1, unicode(kws))
self.assertEqual(kws[0].name, 'this is new')
def test_removing_old_data(self):
self._get_and_insert_keywords('String', '')
self._get_and_insert_keywords('String', '')
newest = self._get_and_insert_keywords('String', '')
newest_again = self._database.fetch_library_keywords('String', '')
self._check_keywords(newest, newest_again)
self.assertEqual(self._database._connection.execute('select count(*) from libraries').fetchone()[0], 1)
self.assertEqual(self._database._connection.execute('select count(*) from keywords').fetchone()[0], len(newest))
def test_unknown_library_keywords_fetch(self):
results = self._database.fetch_library_keywords('Something that is not there', 'at all')
self.assertEqual(len(results), 0)
def test_unknown_library_fetch(self):
self.assertFalse(self._database.library_exists('library', ''))
self._database.insert_library_keywords('library', '', [])
self.assertTrue(self._database.library_exists('library', ''))
def _get_and_insert_keywords(self, library_name, library_arguments):
kws = get_import_result(library_name, library_arguments)
self._database.insert_library_keywords(library_name, library_arguments, kws)
return kws
def _check_keywords(self, originals, from_database):
for k1, k2 in zip(originals, from_database):
self.assertEqual(k1.name, k2.name)
self.assertEqual(k1.doc, k2.doc)
self.assertEqual(k1.arguments, k2.arguments, 'Arguments differ ("%s" != "%s") on keyword %s' %
(k1.arguments, k2.arguments, k1.name))
self.assertEqual(k1.source, k2.source)
self.assertEqual(len(originals), len(from_database))
class TestLibraryDatabase(unittest.TestCase):
def setUp(self):
self._database = LibraryDatabase(':memory:')
self._database.create_database()
def tearDown(self):
self._database.close()
def test_inserting_and_fetching(self):
collections_kws = self._get_and_insert_keywords('Collections', '')
string_kws = self._get_and_insert_keywords('String', '')
builtin_kws = self._get_and_insert_keywords('BuiltIn', '')
collections_kws_from_db = self._database.fetch_library_keywords(
'Collections', '')
string_kws_from_db = self._database.fetch_library_keywords(
'String', '')
builtin_kws_from_db = self._database.fetch_library_keywords(
'BuiltIn', '')
for originals, from_database in [[
collections_kws, collections_kws_from_db
], [string_kws, string_kws_from_db],
[builtin_kws, builtin_kws_from_db]]:
self._check_keywords(originals, from_database)
def test_libkw_doc_format(self):
self._get_and_insert_keywords('testLibText', '')
self._get_and_insert_keywords('testLibRobot', '')
self._get_and_insert_keywords('testLibRest', '')
self._get_and_insert_keywords('testLibHtml', '')
self._get_and_insert_keywords('testLibHtmlClass', '')
text_doc_format_kws = self._database.fetch_library_keywords(
'testLibText', '')
robot_doc_format_kws = self._database.fetch_library_keywords(
'testLibRobot', '')
rest_doc_format_kws = self._database.fetch_library_keywords(
'testLibRest', '')
html1_doc_format_kws = self._database.fetch_library_keywords(
'testLibHtml', '')
html2_doc_format_kws = self._database.fetch_library_keywords(
'testLibHtmlClass', '')
self.assertTrue(
all([kw.doc_format == "TEXT" for kw in text_doc_format_kws]))
self.assertTrue(
all([kw.doc_format == "ROBOT" for kw in robot_doc_format_kws]))
self.assertTrue(
all([kw.doc_format == "REST" for kw in rest_doc_format_kws]))
self.assertTrue(
all([kw.doc_format == "HTML" for kw in html1_doc_format_kws]))
self.assertTrue(
all([kw.doc_format == "HTML" for kw in html2_doc_format_kws]))
def test_valid_doc_format(self):
for format in ("TEXT", "ROBOT", "REST", "HTML"):
orig_kws = [
LibraryKeywordInfo('first', 'doc', format, 'lib.py', ''),
LibraryKeywordInfo('second', 'doc', format, 'lib.py', ''),
LibraryKeywordInfo('third', 'doc', format, 'lib.py', '')
]
self._database.insert_library_keywords('lib.py', 'foo', orig_kws)
kws = self._database.fetch_library_keywords('lib.py', 'foo')
for kw in kws:
self.assertEqual(kw.doc_format, format)
def test_default_doc_format(self):
for format in ("non", "valid", "format", ""):
self._database.insert_library_keywords('lib.py', 'foo', [
LibraryKeywordInfo('this is old', 'doc', format, 'lib.py', '')
])
kws = self._database.fetch_library_keywords('lib.py', 'foo')
self.assertEqual(kws[0].doc_format, "ROBOT")
kws = self._get_and_insert_keywords('String', '')
for kw in kws:
self.assertEqual(kw.doc_format, "ROBOT")
def test_finds_newest_version(self):
self._database.insert_library_keywords(
'lib.py', 'foo',
[LibraryKeywordInfo('this is old', 'doc', 'ROBOT', 'lib.py', '')])
self._database.insert_library_keywords(
'lib.py', 'foo',
[LibraryKeywordInfo('this is new', 'doc', 'ROBOT', 'lib.py', '')])
kws = self._database.fetch_library_keywords('lib.py', 'foo')
self.assertEqual(len(kws), 1, unicode(kws))
self.assertEqual(kws[0].name, 'this is new')
def test_removing_old_data(self):
self._get_and_insert_keywords('String', '')
self._get_and_insert_keywords('String', '')
newest = self._get_and_insert_keywords('String', '')
newest_again = self._database.fetch_library_keywords('String', '')
self._check_keywords(newest, newest_again)
self.assertEqual(
self._database._connection.execute(
'select count(*) from libraries').fetchone()[0], 1)
self.assertEqual(
self._database._connection.execute(
'select count(*) from keywords').fetchone()[0], len(newest))
def test_unknown_library_keywords_fetch(self):
results = self._database.fetch_library_keywords(
'Something that is not there', 'at all')
self.assertEqual(len(results), 0)
def test_unknown_library_fetch(self):
self.assertFalse(self._database.library_exists('library', ''))
self._database.insert_library_keywords('library', '', [])
self.assertTrue(self._database.library_exists('library', ''))
def _get_and_insert_keywords(self, library_name, library_arguments):
kws = get_import_result(library_name, library_arguments)
self._database.insert_library_keywords(library_name, library_arguments,
kws)
return kws
def _check_keywords(self, originals, from_database):
for k1, k2 in zip(originals, from_database):
self.assertEqual(k1.name, k2.name)
self.assertEqual(k1.doc, k2.doc)
self.assertEqual(k1.doc_format, k2.doc_format)
self.assertEqual(
k1.arguments, k2.arguments,
'Arguments differ ("%s" != "%s") on keyword %s' %
(k1.arguments, k2.arguments, k1.name))
self.assertEqual(k1.source, k2.source)
self.assertEqual(len(originals), len(from_database))
def _initiate_database_connection(self):
self._database = LibraryDatabase(self._database_name)
class LibraryManager(Thread):
def __init__(self, database_name, spec_initializer=None):
self._database_name = database_name
self._database = None
self._messages = Queue.Queue()
self._spec_initializer = spec_initializer or SpecInitializer()
Thread.__init__(self)
self.setDaemon(True)
def run(self):
self._initiate_database_connection()
while True:
try:
if not self._handle_message():
break
except Exception as err:
msg = 'Library import handling threw an unexpected exception'
RideLogException(message=msg, exception=err, level='WARN').publish()
self._database.close()
def _initiate_database_connection(self):
self._database = LibraryDatabase(self._database_name)
def get_new_connection_to_library_database(self):
library_database = LibraryDatabase(self._database_name)
if self._database_name == ':memory:':
# In memory database does not point to the right place..
# this is here for unit tests..
library_database.create_database()
return library_database
def _handle_message(self):
message = self._messages.get()
if not message:
return False
msg_type = message[0]
if msg_type == 'fetch':
self._handle_fetch_keywords_message(message)
elif msg_type == 'insert':
self._handle_insert_keywords_message(message)
elif msg_type == 'create':
self._database.create_database()
return True
def _handle_fetch_keywords_message(self, message):
_, library_name, library_args, callback = message
keywords = self._fetch_keywords(library_name, library_args)
self._update_database_and_call_callback_if_needed(
(library_name, library_args), keywords, callback)
def _fetch_keywords(self, library_name, library_args):
try:
path = get_path(
library_name.replace('/', os.sep), os.path.abspath('.'))
return get_import_result(path, library_args)
except Exception as err:
try:
print('FAILED', library_name, err)
except IOError:
pass
kws = self._spec_initializer.init_from_spec(library_name)
if not kws:
msg = 'Importing test library "%s" failed' % library_name
RideLogException(
message=msg, exception=err, level='WARN').publish()
return kws
def _handle_insert_keywords_message(self, message):
_, library_name, library_args, result_queue = message
keywords = self._fetch_keywords(library_name, library_args)
self._insert(library_name, library_args, keywords,
lambda res: result_queue.put(res, timeout=3))
def _insert(self, library_name, library_args, keywords, callback):
self._database.insert_library_keywords(
library_name, library_args, keywords or [])
self._call(callback, keywords)
def _update_database_and_call_callback_if_needed(
self, library_key, keywords, callback):
db_keywords = self._database.fetch_library_keywords(*library_key)
try:
if not db_keywords or self._keywords_differ(keywords, db_keywords):
self._insert(
library_key[0], library_key[1], keywords, callback)
else:
self._database.update_library_timestamp(*library_key)
except OperationalError:
pass
def _call(self, callback, *args):
try:
callback(*args)
except Exception as err:
msg = 'Library import callback threw an unexpected exception'
RideLogException(message=msg, exception=err, level='WARN').publish()
def fetch_keywords(self, library_name, library_args, callback):
self._messages.put(('fetch', library_name, library_args, callback),
timeout=3)
def get_and_insert_keywords(self, library_name, library_args):
result_queue = Queue.Queue(maxsize=1)
self._messages.put(
('insert', library_name, library_args, result_queue), timeout=3)
try:
return result_queue.get(timeout=5)
except Queue.Empty as e:
RideLogMessage(u'Failed to read keywords from library db: {}'
.format(unicode(e))).publish()
return []
def create_database(self):
self._messages.put(('create',), timeout=3)
def stop(self):
self._messages.put(False, timeout=3)
def _keywords_differ(self, keywords1, keywords2):
if keywords1 != keywords2 and None in (keywords1, keywords2):
return True
if len(keywords1) != len(keywords2):
return True
for k1, k2 in zip(keywords1, keywords2):
if k1.name != k2.name:
return True
if k1.doc != k2.doc:
return True
# if k1.doc_format != k2.doc_format:
# return True
if k1.arguments != k2.arguments:
return True
if k1.source != k2.source:
return True
return False
