class TestLibraryDatabase(unittest.TestCase): def setUp(self): self._database = LibraryDatabase(':memory:') self._database.create_database() def tearDown(self): self._database.close() def test_inserting_and_fetching(self): collections_kws = self._get_and_insert_keywords('Collections', '') string_kws = self._get_and_insert_keywords('String', '') builtin_kws = self._get_and_insert_keywords('BuiltIn', '') collections_kws_from_db = self._database.fetch_library_keywords('Collections', '') string_kws_from_db = self._database.fetch_library_keywords('String', '') builtin_kws_from_db = self._database.fetch_library_keywords('BuiltIn', '') for originals, from_database in [[collections_kws, collections_kws_from_db], [string_kws, string_kws_from_db], [builtin_kws, builtin_kws_from_db]]: self._check_keywords(originals, from_database) def test_finds_newest_version(self): self._database.insert_library_keywords('lib.py', 'foo', [LibraryKeywordInfo('this is old', 'doc', 'lib.py', '')]) self._database.insert_library_keywords('lib.py', 'foo', [LibraryKeywordInfo('this is new', 'doc', 'lib.py', '')]) kws = self._database.fetch_library_keywords('lib.py', 'foo') self.assertEqual(len(kws), 1, unicode(kws)) self.assertEqual(kws[0].name, 'this is new') def test_removing_old_data(self): self._get_and_insert_keywords('String', '') self._get_and_insert_keywords('String', '') newest = self._get_and_insert_keywords('String', '') newest_again = self._database.fetch_library_keywords('String', '') self._check_keywords(newest, newest_again) self.assertEqual(self._database._connection.execute('select count(*) from libraries').fetchone()[0], 1) self.assertEqual(self._database._connection.execute('select count(*) from keywords').fetchone()[0], len(newest)) def test_unknown_library_keywords_fetch(self): results = self._database.fetch_library_keywords('Something that is not there', 'at all') self.assertEqual(len(results), 0) def test_unknown_library_fetch(self): self.assertFalse(self._database.library_exists('library', '')) self._database.insert_library_keywords('library', '', []) self.assertTrue(self._database.library_exists('library', '')) def _get_and_insert_keywords(self, library_name, library_arguments): kws = get_import_result(library_name, library_arguments) self._database.insert_library_keywords(library_name, library_arguments, kws) return kws def _check_keywords(self, originals, from_database): for k1, k2 in zip(originals, from_database): self.assertEqual(k1.name, k2.name) self.assertEqual(k1.doc, k2.doc) self.assertEqual(k1.arguments, k2.arguments, 'Arguments differ ("%s" != "%s") on keyword %s' % (k1.arguments, k2.arguments, k1.name)) self.assertEqual(k1.source, k2.source) self.assertEqual(len(originals), len(from_database))
class LibraryManager(Thread): def __init__(self, database_name, spec_initializer=None): self._database_name = database_name self._database = None self._messages = Queue.Queue() self._spec_initializer = spec_initializer or SpecInitializer() Thread.__init__(self) self.setDaemon(True) def run(self): self._initiate_database_connection() while True: try: if not self._handle_message(): break except Exception as err: msg = 'Library import handling threw an unexpected exception' RideLogException(message=msg, exception=err, level='WARN').publish() self._database.close() def _initiate_database_connection(self): self._database = LibraryDatabase(self._database_name) def get_new_connection_to_library_database(self): library_database = LibraryDatabase(self._database_name) if self._database_name == ':memory:': # In memory database does not point to the right place.. # this is here for unit tests.. library_database.create_database() return library_database def _handle_message(self): message = self._messages.get() if not message: return False msg_type = message[0] if msg_type == 'fetch': self._handle_fetch_keywords_message(message) elif msg_type == 'insert': self._handle_insert_keywords_message(message) elif msg_type == 'create': self._database.create_database() return True def _handle_fetch_keywords_message(self, message): _, library_name, library_args, callback = message keywords = self._fetch_keywords(library_name, library_args) self._update_database_and_call_callback_if_needed( (library_name, library_args), keywords, callback) def _fetch_keywords(self, library_name, library_args): try: path = get_path(library_name.replace('/', os.sep), os.path.abspath('.')) return get_import_result(path, library_args) except Exception as err: try: print('FAILED', library_name, err) except IOError: pass kws = self._spec_initializer.init_from_spec(library_name) if not kws: msg = 'Importing test library "%s" failed' % library_name RideLogException(message=msg, exception=err, level='WARN').publish() return kws def _handle_insert_keywords_message(self, message): _, library_name, library_args, result_queue = message keywords = self._fetch_keywords(library_name, library_args) self._insert(library_name, library_args, keywords, lambda res: result_queue.put(res, timeout=3)) def _insert(self, library_name, library_args, keywords, callback): self._database.insert_library_keywords(library_name, library_args, keywords or []) self._call(callback, keywords) def _update_database_and_call_callback_if_needed(self, library_key, keywords, callback): db_keywords = self._database.fetch_library_keywords(*library_key) try: if not db_keywords or self._keywords_differ(keywords, db_keywords): self._insert(library_key[0], library_key[1], keywords, callback) else: self._database.update_library_timestamp(*library_key) except OperationalError: pass def _call(self, callback, *args): try: callback(*args) except Exception as err: msg = 'Library import callback threw an unexpected exception' RideLogException(message=msg, exception=err, level='WARN').publish() def fetch_keywords(self, library_name, library_args, callback): self._messages.put(('fetch', library_name, library_args, callback), timeout=3) def get_and_insert_keywords(self, library_name, library_args): result_queue = Queue.Queue(maxsize=1) self._messages.put( ('insert', library_name, library_args, result_queue), timeout=3) try: return result_queue.get(timeout=5) except Queue.Empty as e: RideLogMessage( u'Failed to read keywords from library db: {}'.format( unicode(e))).publish() return [] def create_database(self): self._messages.put(('create', ), timeout=3) def stop(self): self._messages.put(False, timeout=3) def _keywords_differ(self, keywords1, keywords2): if keywords1 != keywords2 and None in (keywords1, keywords2): return True if len(keywords1) != len(keywords2): return True for k1, k2 in zip(keywords1, keywords2): if k1.name != k2.name: return True if k1.doc != k2.doc: return True # if k1.doc_format != k2.doc_format: # return True if k1.arguments != k2.arguments: return True if k1.source != k2.source: return True return False
class TestLibraryDatabase(unittest.TestCase): def setUp(self): self._database = LibraryDatabase(':memory:') self._database.create_database() def tearDown(self): self._database.close() def test_inserting_and_fetching(self): collections_kws = self._get_and_insert_keywords('Collections', '') string_kws = self._get_and_insert_keywords('String', '') builtin_kws = self._get_and_insert_keywords('BuiltIn', '') collections_kws_from_db = self._database.fetch_library_keywords( 'Collections', '') string_kws_from_db = self._database.fetch_library_keywords( 'String', '') builtin_kws_from_db = self._database.fetch_library_keywords( 'BuiltIn', '') for originals, from_database in [[ collections_kws, collections_kws_from_db ], [string_kws, string_kws_from_db], [builtin_kws, builtin_kws_from_db]]: self._check_keywords(originals, from_database) def test_finds_newest_version(self): self._database.insert_library_keywords( 'lib.py', 'foo', [LibraryKeywordInfo('this is old', 'doc', 'lib.py', '')]) self._database.insert_library_keywords( 'lib.py', 'foo', [LibraryKeywordInfo('this is new', 'doc', 'lib.py', '')]) kws = self._database.fetch_library_keywords('lib.py', 'foo') self.assertEqual(len(kws), 1, unicode(kws)) self.assertEqual(kws[0].name, 'this is new') def test_removing_old_data(self): self._get_and_insert_keywords('String', '') self._get_and_insert_keywords('String', '') newest = self._get_and_insert_keywords('String', '') newest_again = self._database.fetch_library_keywords('String', '') self._check_keywords(newest, newest_again) self.assertEqual( self._database._connection.execute( 'select count(*) from libraries').fetchone()[0], 1) self.assertEqual( self._database._connection.execute( 'select count(*) from keywords').fetchone()[0], len(newest)) def test_unknown_library_keywords_fetch(self): results = self._database.fetch_library_keywords( 'Something that is not there', 'at all') self.assertEqual(len(results), 0) def test_unknown_library_fetch(self): self.assertFalse(self._database.library_exists('library', '')) self._database.insert_library_keywords('library', '', []) self.assertTrue(self._database.library_exists('library', '')) def _get_and_insert_keywords(self, library_name, library_arguments): kws = get_import_result(library_name, library_arguments) self._database.insert_library_keywords(library_name, library_arguments, kws) return kws def _check_keywords(self, originals, from_database): for k1, k2 in zip(originals, from_database): self.assertEqual(k1.name, k2.name) self.assertEqual(k1.doc, k2.doc) self.assertEqual( k1.arguments, k2.arguments, 'Arguments differ ("%s" != "%s") on keyword %s' % (k1.arguments, k2.arguments, k1.name)) self.assertEqual(k1.source, k2.source) self.assertEqual(len(originals), len(from_database))
class TestLibraryDatabase(unittest.TestCase): def setUp(self): self._database = LibraryDatabase(':memory:') self._database.create_database() def tearDown(self): self._database.close() def test_inserting_and_fetching(self): collections_kws = self._get_and_insert_keywords('Collections', '') string_kws = self._get_and_insert_keywords('String', '') builtin_kws = self._get_and_insert_keywords('BuiltIn', '') collections_kws_from_db = self._database.fetch_library_keywords( 'Collections', '') string_kws_from_db = self._database.fetch_library_keywords( 'String', '') builtin_kws_from_db = self._database.fetch_library_keywords( 'BuiltIn', '') for originals, from_database in [[ collections_kws, collections_kws_from_db ], [string_kws, string_kws_from_db], [builtin_kws, builtin_kws_from_db]]: self._check_keywords(originals, from_database) def test_libkw_doc_format(self): self._get_and_insert_keywords('testLibText', '') self._get_and_insert_keywords('testLibRobot', '') self._get_and_insert_keywords('testLibRest', '') self._get_and_insert_keywords('testLibHtml', '') self._get_and_insert_keywords('testLibHtmlClass', '') text_doc_format_kws = self._database.fetch_library_keywords( 'testLibText', '') robot_doc_format_kws = self._database.fetch_library_keywords( 'testLibRobot', '') rest_doc_format_kws = self._database.fetch_library_keywords( 'testLibRest', '') html1_doc_format_kws = self._database.fetch_library_keywords( 'testLibHtml', '') html2_doc_format_kws = self._database.fetch_library_keywords( 'testLibHtmlClass', '') self.assertTrue( all([kw.doc_format == "TEXT" for kw in text_doc_format_kws])) self.assertTrue( all([kw.doc_format == "ROBOT" for kw in robot_doc_format_kws])) self.assertTrue( all([kw.doc_format == "REST" for kw in rest_doc_format_kws])) self.assertTrue( all([kw.doc_format == "HTML" for kw in html1_doc_format_kws])) self.assertTrue( all([kw.doc_format == "HTML" for kw in html2_doc_format_kws])) def test_valid_doc_format(self): for format in ("TEXT", "ROBOT", "REST", "HTML"): orig_kws = [ LibraryKeywordInfo('first', 'doc', format, 'lib.py', ''), LibraryKeywordInfo('second', 'doc', format, 'lib.py', ''), LibraryKeywordInfo('third', 'doc', format, 'lib.py', '') ] self._database.insert_library_keywords('lib.py', 'foo', orig_kws) kws = self._database.fetch_library_keywords('lib.py', 'foo') for kw in kws: self.assertEqual(kw.doc_format, format) def test_default_doc_format(self): for format in ("non", "valid", "format", ""): self._database.insert_library_keywords('lib.py', 'foo', [ LibraryKeywordInfo('this is old', 'doc', format, 'lib.py', '') ]) kws = self._database.fetch_library_keywords('lib.py', 'foo') self.assertEqual(kws[0].doc_format, "ROBOT") kws = self._get_and_insert_keywords('String', '') for kw in kws: self.assertEqual(kw.doc_format, "ROBOT") def test_finds_newest_version(self): self._database.insert_library_keywords( 'lib.py', 'foo', [LibraryKeywordInfo('this is old', 'doc', 'ROBOT', 'lib.py', '')]) self._database.insert_library_keywords( 'lib.py', 'foo', [LibraryKeywordInfo('this is new', 'doc', 'ROBOT', 'lib.py', '')]) kws = self._database.fetch_library_keywords('lib.py', 'foo') self.assertEqual(len(kws), 1, unicode(kws)) self.assertEqual(kws[0].name, 'this is new') def test_removing_old_data(self): self._get_and_insert_keywords('String', '') self._get_and_insert_keywords('String', '') newest = self._get_and_insert_keywords('String', '') newest_again = self._database.fetch_library_keywords('String', '') self._check_keywords(newest, newest_again) self.assertEqual( self._database._connection.execute( 'select count(*) from libraries').fetchone()[0], 1) self.assertEqual( self._database._connection.execute( 'select count(*) from keywords').fetchone()[0], len(newest)) def test_unknown_library_keywords_fetch(self): results = self._database.fetch_library_keywords( 'Something that is not there', 'at all') self.assertEqual(len(results), 0) def test_unknown_library_fetch(self): self.assertFalse(self._database.library_exists('library', '')) self._database.insert_library_keywords('library', '', []) self.assertTrue(self._database.library_exists('library', '')) def _get_and_insert_keywords(self, library_name, library_arguments): kws = get_import_result(library_name, library_arguments) self._database.insert_library_keywords(library_name, library_arguments, kws) return kws def _check_keywords(self, originals, from_database): for k1, k2 in zip(originals, from_database): self.assertEqual(k1.name, k2.name) self.assertEqual(k1.doc, k2.doc) self.assertEqual(k1.doc_format, k2.doc_format) self.assertEqual( k1.arguments, k2.arguments, 'Arguments differ ("%s" != "%s") on keyword %s' % (k1.arguments, k2.arguments, k1.name)) self.assertEqual(k1.source, k2.source) self.assertEqual(len(originals), len(from_database))
class LibraryManager(Thread): def __init__(self, database_name, spec_initializer=None): self._database_name = database_name self._database = None self._messages = Queue.Queue() self._spec_initializer = spec_initializer or SpecInitializer() Thread.__init__(self) self.setDaemon(True) def run(self): self._initiate_database_connection() while True: try: if not self._handle_message(): break except Exception as err: msg = 'Library import handling threw an unexpected exception' RideLogException(message=msg, exception=err, level='WARN').publish() self._database.close() def _initiate_database_connection(self): self._database = LibraryDatabase(self._database_name) def get_new_connection_to_library_database(self): library_database = LibraryDatabase(self._database_name) if self._database_name == ':memory:': # In memory database does not point to the right place.. # this is here for unit tests.. library_database.create_database() return library_database def _handle_message(self): message = self._messages.get() if not message: return False msg_type = message[0] if msg_type == 'fetch': self._handle_fetch_keywords_message(message) elif msg_type == 'insert': self._handle_insert_keywords_message(message) elif msg_type == 'create': self._database.create_database() return True def _handle_fetch_keywords_message(self, message): _, library_name, library_args, callback = message keywords = self._fetch_keywords(library_name, library_args) self._update_database_and_call_callback_if_needed( (library_name, library_args), keywords, callback) def _fetch_keywords(self, library_name, library_args): try: path = get_path( library_name.replace('/', os.sep), os.path.abspath('.')) return get_import_result(path, library_args) except Exception as err: try: print('FAILED', library_name, err) except IOError: pass kws = self._spec_initializer.init_from_spec(library_name) if not kws: msg = 'Importing test library "%s" failed' % library_name RideLogException( message=msg, exception=err, level='WARN').publish() return kws def _handle_insert_keywords_message(self, message): _, library_name, library_args, result_queue = message keywords = self._fetch_keywords(library_name, library_args) self._insert(library_name, library_args, keywords, lambda res: result_queue.put(res, timeout=3)) def _insert(self, library_name, library_args, keywords, callback): self._database.insert_library_keywords( library_name, library_args, keywords or []) self._call(callback, keywords) def _update_database_and_call_callback_if_needed( self, library_key, keywords, callback): db_keywords = self._database.fetch_library_keywords(*library_key) try: if not db_keywords or self._keywords_differ(keywords, db_keywords): self._insert( library_key[0], library_key[1], keywords, callback) else: self._database.update_library_timestamp(*library_key) except OperationalError: pass def _call(self, callback, *args): try: callback(*args) except Exception as err: msg = 'Library import callback threw an unexpected exception' RideLogException(message=msg, exception=err, level='WARN').publish() def fetch_keywords(self, library_name, library_args, callback): self._messages.put(('fetch', library_name, library_args, callback), timeout=3) def get_and_insert_keywords(self, library_name, library_args): result_queue = Queue.Queue(maxsize=1) self._messages.put( ('insert', library_name, library_args, result_queue), timeout=3) try: return result_queue.get(timeout=5) except Queue.Empty as e: RideLogMessage(u'Failed to read keywords from library db: {}' .format(unicode(e))).publish() return [] def create_database(self): self._messages.put(('create',), timeout=3) def stop(self): self._messages.put(False, timeout=3) def _keywords_differ(self, keywords1, keywords2): if keywords1 != keywords2 and None in (keywords1, keywords2): return True if len(keywords1) != len(keywords2): return True for k1, k2 in zip(keywords1, keywords2): if k1.name != k2.name: return True if k1.doc != k2.doc: return True # if k1.doc_format != k2.doc_format: # return True if k1.arguments != k2.arguments: return True if k1.source != k2.source: return True return False