Ejemplo n.º 1
0
 def get_new_connection_to_library_database(self):
     library_database = LibraryDatabase(self._database_name)
     if self._database_name == ':memory:':
         # In memory database does not point to the right place..
         # this is here for unit tests..
         library_database.create_database()
     return library_database
Ejemplo n.º 2
0
 def get_new_connection_to_library_database(self):
     library_database = LibraryDatabase(self._database_name)
     if self._database_name == ':memory:':
         # In memory database does not point to the right place..
         # this is here for unit tests..
         library_database.create_database()
     return library_database
Ejemplo n.º 3
0
 def _initiate_database_connection(self):
     self._database = LibraryDatabase(self._database_name)
Ejemplo n.º 4
0
class LibraryManager(Thread):
    def __init__(self, database_name, spec_initializer=None):
        self._database_name = database_name
        self._database = None
        self._messages = Queue.Queue()
        self._spec_initializer = spec_initializer or SpecInitializer()
        Thread.__init__(self)
        self.setDaemon(True)

    def run(self):
        self._initiate_database_connection()
        while True:
            try:
                if not self._handle_message():
                    break
            except Exception as err:
                msg = 'Library import handling threw an unexpected exception'
                RideLogException(message=msg, exception=err,
                                 level='WARN').publish()
        self._database.close()

    def _initiate_database_connection(self):
        self._database = LibraryDatabase(self._database_name)

    def get_new_connection_to_library_database(self):
        library_database = LibraryDatabase(self._database_name)
        if self._database_name == ':memory:':
            # In memory database does not point to the right place..
            # this is here for unit tests..
            library_database.create_database()
        return library_database

    def _handle_message(self):
        message = self._messages.get()
        if not message:
            return False
        msg_type = message[0]
        if msg_type == 'fetch':
            self._handle_fetch_keywords_message(message)
        elif msg_type == 'insert':
            self._handle_insert_keywords_message(message)
        elif msg_type == 'create':
            self._database.create_database()
        return True

    def _handle_fetch_keywords_message(self, message):
        _, library_name, library_args, callback = message
        keywords = self._fetch_keywords(library_name, library_args)
        self._update_database_and_call_callback_if_needed(
            (library_name, library_args), keywords, callback)

    def _fetch_keywords(self, library_name, library_args):
        try:
            path = get_path(library_name.replace('/', os.sep),
                            os.path.abspath('.'))
            return get_import_result(path, library_args)
        except Exception as err:
            try:
                print('FAILED', library_name, err)
            except IOError:
                pass
            kws = self._spec_initializer.init_from_spec(library_name)
            if not kws:
                msg = 'Importing test library "%s" failed' % library_name
                RideLogException(message=msg, exception=err,
                                 level='WARN').publish()
            return kws

    def _handle_insert_keywords_message(self, message):
        _, library_name, library_args, result_queue = message
        keywords = self._fetch_keywords(library_name, library_args)
        self._insert(library_name, library_args, keywords,
                     lambda res: result_queue.put(res, timeout=3))

    def _insert(self, library_name, library_args, keywords, callback):
        self._database.insert_library_keywords(library_name, library_args,
                                               keywords or [])
        self._call(callback, keywords)

    def _update_database_and_call_callback_if_needed(self, library_key,
                                                     keywords, callback):
        db_keywords = self._database.fetch_library_keywords(*library_key)
        try:
            if not db_keywords or self._keywords_differ(keywords, db_keywords):
                self._insert(library_key[0], library_key[1], keywords,
                             callback)
            else:
                self._database.update_library_timestamp(*library_key)
        except OperationalError:
            pass

    def _call(self, callback, *args):
        try:
            callback(*args)
        except Exception as err:
            msg = 'Library import callback threw an unexpected exception'
            RideLogException(message=msg, exception=err,
                             level='WARN').publish()

    def fetch_keywords(self, library_name, library_args, callback):
        self._messages.put(('fetch', library_name, library_args, callback),
                           timeout=3)

    def get_and_insert_keywords(self, library_name, library_args):
        result_queue = Queue.Queue(maxsize=1)
        self._messages.put(
            ('insert', library_name, library_args, result_queue), timeout=3)
        try:
            return result_queue.get(timeout=5)
        except Queue.Empty as e:
            RideLogMessage(
                u'Failed to read keywords from library db: {}'.format(
                    unicode(e))).publish()
            return []

    def create_database(self):
        self._messages.put(('create', ), timeout=3)

    def stop(self):
        self._messages.put(False, timeout=3)

    def _keywords_differ(self, keywords1, keywords2):
        if keywords1 != keywords2 and None in (keywords1, keywords2):
            return True
        if len(keywords1) != len(keywords2):
            return True
        for k1, k2 in zip(keywords1, keywords2):
            if k1.name != k2.name:
                return True
            if k1.doc != k2.doc:
                return True
            # if k1.doc_format != k2.doc_format:
            #     return True
            if k1.arguments != k2.arguments:
                return True
            if k1.source != k2.source:
                return True
        return False
Ejemplo n.º 5
0
 def setUp(self):
     self._database = LibraryDatabase(':memory:')
     self._database.create_database()
Ejemplo n.º 6
0
class TestLibraryDatabase(unittest.TestCase):
    def setUp(self):
        self._database = LibraryDatabase(':memory:')
        self._database.create_database()

    def tearDown(self):
        self._database.close()

    def test_inserting_and_fetching(self):
        collections_kws = self._get_and_insert_keywords('Collections', '')
        string_kws = self._get_and_insert_keywords('String', '')
        builtin_kws = self._get_and_insert_keywords('BuiltIn', '')
        collections_kws_from_db = self._database.fetch_library_keywords(
            'Collections', '')
        string_kws_from_db = self._database.fetch_library_keywords(
            'String', '')
        builtin_kws_from_db = self._database.fetch_library_keywords(
            'BuiltIn', '')
        for originals, from_database in [[
                collections_kws, collections_kws_from_db
        ], [string_kws, string_kws_from_db],
                                         [builtin_kws, builtin_kws_from_db]]:
            self._check_keywords(originals, from_database)

    def test_finds_newest_version(self):
        self._database.insert_library_keywords(
            'lib.py', 'foo',
            [LibraryKeywordInfo('this is old', 'doc', 'lib.py', '')])
        self._database.insert_library_keywords(
            'lib.py', 'foo',
            [LibraryKeywordInfo('this is new', 'doc', 'lib.py', '')])
        kws = self._database.fetch_library_keywords('lib.py', 'foo')
        self.assertEqual(len(kws), 1, unicode(kws))
        self.assertEqual(kws[0].name, 'this is new')

    def test_removing_old_data(self):
        self._get_and_insert_keywords('String', '')
        self._get_and_insert_keywords('String', '')
        newest = self._get_and_insert_keywords('String', '')
        newest_again = self._database.fetch_library_keywords('String', '')
        self._check_keywords(newest, newest_again)
        self.assertEqual(
            self._database._connection.execute(
                'select count(*) from libraries').fetchone()[0], 1)
        self.assertEqual(
            self._database._connection.execute(
                'select count(*) from keywords').fetchone()[0], len(newest))

    def test_unknown_library_keywords_fetch(self):
        results = self._database.fetch_library_keywords(
            'Something that is not there', 'at all')
        self.assertEqual(len(results), 0)

    def test_unknown_library_fetch(self):
        self.assertFalse(self._database.library_exists('library', ''))
        self._database.insert_library_keywords('library', '', [])
        self.assertTrue(self._database.library_exists('library', ''))

    def _get_and_insert_keywords(self, library_name, library_arguments):
        kws = get_import_result(library_name, library_arguments)
        self._database.insert_library_keywords(library_name, library_arguments,
                                               kws)
        return kws

    def _check_keywords(self, originals, from_database):
        for k1, k2 in zip(originals, from_database):
            self.assertEqual(k1.name, k2.name)
            self.assertEqual(k1.doc, k2.doc)
            self.assertEqual(
                k1.arguments, k2.arguments,
                'Arguments differ ("%s" != "%s") on keyword %s' %
                (k1.arguments, k2.arguments, k1.name))
            self.assertEqual(k1.source, k2.source)
        self.assertEqual(len(originals), len(from_database))
Ejemplo n.º 7
0
 def setUp(self):
     self._database = LibraryDatabase(':memory:')
     self._database.create_database()
Ejemplo n.º 8
0
class TestLibraryDatabase(unittest.TestCase):

    def setUp(self):
        self._database = LibraryDatabase(':memory:')
        self._database.create_database()

    def tearDown(self):
        self._database.close()

    def test_inserting_and_fetching(self):
        collections_kws = self._get_and_insert_keywords('Collections', '')
        string_kws = self._get_and_insert_keywords('String', '')
        builtin_kws = self._get_and_insert_keywords('BuiltIn', '')
        collections_kws_from_db = self._database.fetch_library_keywords('Collections', '')
        string_kws_from_db = self._database.fetch_library_keywords('String', '')
        builtin_kws_from_db = self._database.fetch_library_keywords('BuiltIn', '')
        for originals, from_database in [[collections_kws, collections_kws_from_db],
                                         [string_kws, string_kws_from_db],
                                         [builtin_kws, builtin_kws_from_db]]:
            self._check_keywords(originals, from_database)

    def test_finds_newest_version(self):
        self._database.insert_library_keywords('lib.py', 'foo', [LibraryKeywordInfo('this is old', 'doc', 'lib.py', '')])
        self._database.insert_library_keywords('lib.py', 'foo', [LibraryKeywordInfo('this is new', 'doc', 'lib.py', '')])
        kws = self._database.fetch_library_keywords('lib.py', 'foo')
        self.assertEqual(len(kws), 1, unicode(kws))
        self.assertEqual(kws[0].name, 'this is new')

    def test_removing_old_data(self):
        self._get_and_insert_keywords('String', '')
        self._get_and_insert_keywords('String', '')
        newest = self._get_and_insert_keywords('String', '')
        newest_again = self._database.fetch_library_keywords('String', '')
        self._check_keywords(newest, newest_again)
        self.assertEqual(self._database._connection.execute('select count(*) from libraries').fetchone()[0], 1)
        self.assertEqual(self._database._connection.execute('select count(*) from keywords').fetchone()[0], len(newest))

    def test_unknown_library_keywords_fetch(self):
        results = self._database.fetch_library_keywords('Something that is not there', 'at all')
        self.assertEqual(len(results), 0)

    def test_unknown_library_fetch(self):
        self.assertFalse(self._database.library_exists('library', ''))
        self._database.insert_library_keywords('library', '', [])
        self.assertTrue(self._database.library_exists('library', ''))

    def _get_and_insert_keywords(self, library_name, library_arguments):
        kws = get_import_result(library_name, library_arguments)
        self._database.insert_library_keywords(library_name, library_arguments, kws)
        return kws

    def _check_keywords(self, originals, from_database):
        for k1, k2 in zip(originals, from_database):
            self.assertEqual(k1.name, k2.name)
            self.assertEqual(k1.doc, k2.doc)
            self.assertEqual(k1.arguments, k2.arguments, 'Arguments differ ("%s" != "%s") on keyword %s' %
                                                         (k1.arguments, k2.arguments, k1.name))
            self.assertEqual(k1.source, k2.source)
        self.assertEqual(len(originals), len(from_database))
class TestLibraryDatabase(unittest.TestCase):
    def setUp(self):
        self._database = LibraryDatabase(':memory:')
        self._database.create_database()

    def tearDown(self):
        self._database.close()

    def test_inserting_and_fetching(self):
        collections_kws = self._get_and_insert_keywords('Collections', '')
        string_kws = self._get_and_insert_keywords('String', '')
        builtin_kws = self._get_and_insert_keywords('BuiltIn', '')
        collections_kws_from_db = self._database.fetch_library_keywords(
            'Collections', '')
        string_kws_from_db = self._database.fetch_library_keywords(
            'String', '')
        builtin_kws_from_db = self._database.fetch_library_keywords(
            'BuiltIn', '')
        for originals, from_database in [[
                collections_kws, collections_kws_from_db
        ], [string_kws, string_kws_from_db],
                                         [builtin_kws, builtin_kws_from_db]]:
            self._check_keywords(originals, from_database)

    def test_libkw_doc_format(self):
        self._get_and_insert_keywords('testLibText', '')
        self._get_and_insert_keywords('testLibRobot', '')
        self._get_and_insert_keywords('testLibRest', '')
        self._get_and_insert_keywords('testLibHtml', '')
        self._get_and_insert_keywords('testLibHtmlClass', '')

        text_doc_format_kws = self._database.fetch_library_keywords(
            'testLibText', '')
        robot_doc_format_kws = self._database.fetch_library_keywords(
            'testLibRobot', '')
        rest_doc_format_kws = self._database.fetch_library_keywords(
            'testLibRest', '')
        html1_doc_format_kws = self._database.fetch_library_keywords(
            'testLibHtml', '')
        html2_doc_format_kws = self._database.fetch_library_keywords(
            'testLibHtmlClass', '')

        self.assertTrue(
            all([kw.doc_format == "TEXT" for kw in text_doc_format_kws]))
        self.assertTrue(
            all([kw.doc_format == "ROBOT" for kw in robot_doc_format_kws]))
        self.assertTrue(
            all([kw.doc_format == "REST" for kw in rest_doc_format_kws]))
        self.assertTrue(
            all([kw.doc_format == "HTML" for kw in html1_doc_format_kws]))
        self.assertTrue(
            all([kw.doc_format == "HTML" for kw in html2_doc_format_kws]))

    def test_valid_doc_format(self):
        for format in ("TEXT", "ROBOT", "REST", "HTML"):
            orig_kws = [
                LibraryKeywordInfo('first', 'doc', format, 'lib.py', ''),
                LibraryKeywordInfo('second', 'doc', format, 'lib.py', ''),
                LibraryKeywordInfo('third', 'doc', format, 'lib.py', '')
            ]

            self._database.insert_library_keywords('lib.py', 'foo', orig_kws)
            kws = self._database.fetch_library_keywords('lib.py', 'foo')
            for kw in kws:
                self.assertEqual(kw.doc_format, format)

    def test_default_doc_format(self):
        for format in ("non", "valid", "format", ""):
            self._database.insert_library_keywords('lib.py', 'foo', [
                LibraryKeywordInfo('this is old', 'doc', format, 'lib.py', '')
            ])
            kws = self._database.fetch_library_keywords('lib.py', 'foo')
            self.assertEqual(kws[0].doc_format, "ROBOT")

        kws = self._get_and_insert_keywords('String', '')
        for kw in kws:
            self.assertEqual(kw.doc_format, "ROBOT")

    def test_finds_newest_version(self):
        self._database.insert_library_keywords(
            'lib.py', 'foo',
            [LibraryKeywordInfo('this is old', 'doc', 'ROBOT', 'lib.py', '')])
        self._database.insert_library_keywords(
            'lib.py', 'foo',
            [LibraryKeywordInfo('this is new', 'doc', 'ROBOT', 'lib.py', '')])
        kws = self._database.fetch_library_keywords('lib.py', 'foo')
        self.assertEqual(len(kws), 1, unicode(kws))
        self.assertEqual(kws[0].name, 'this is new')

    def test_removing_old_data(self):
        self._get_and_insert_keywords('String', '')
        self._get_and_insert_keywords('String', '')
        newest = self._get_and_insert_keywords('String', '')
        newest_again = self._database.fetch_library_keywords('String', '')
        self._check_keywords(newest, newest_again)
        self.assertEqual(
            self._database._connection.execute(
                'select count(*) from libraries').fetchone()[0], 1)
        self.assertEqual(
            self._database._connection.execute(
                'select count(*) from keywords').fetchone()[0], len(newest))

    def test_unknown_library_keywords_fetch(self):
        results = self._database.fetch_library_keywords(
            'Something that is not there', 'at all')
        self.assertEqual(len(results), 0)

    def test_unknown_library_fetch(self):
        self.assertFalse(self._database.library_exists('library', ''))
        self._database.insert_library_keywords('library', '', [])
        self.assertTrue(self._database.library_exists('library', ''))

    def _get_and_insert_keywords(self, library_name, library_arguments):
        kws = get_import_result(library_name, library_arguments)
        self._database.insert_library_keywords(library_name, library_arguments,
                                               kws)
        return kws

    def _check_keywords(self, originals, from_database):
        for k1, k2 in zip(originals, from_database):
            self.assertEqual(k1.name, k2.name)
            self.assertEqual(k1.doc, k2.doc)
            self.assertEqual(k1.doc_format, k2.doc_format)
            self.assertEqual(
                k1.arguments, k2.arguments,
                'Arguments differ ("%s" != "%s") on keyword %s' %
                (k1.arguments, k2.arguments, k1.name))
            self.assertEqual(k1.source, k2.source)
        self.assertEqual(len(originals), len(from_database))
Ejemplo n.º 10
0
 def _initiate_database_connection(self):
     self._database = LibraryDatabase(self._database_name)
Ejemplo n.º 11
0
class LibraryManager(Thread):

    def __init__(self, database_name, spec_initializer=None):
        self._database_name = database_name
        self._database = None
        self._messages = Queue.Queue()
        self._spec_initializer = spec_initializer or SpecInitializer()
        Thread.__init__(self)
        self.setDaemon(True)

    def run(self):
        self._initiate_database_connection()
        while True:
            try:
                if not self._handle_message():
                    break
            except Exception as err:
                msg = 'Library import handling threw an unexpected exception'
                RideLogException(message=msg, exception=err, level='WARN').publish()
        self._database.close()

    def _initiate_database_connection(self):
        self._database = LibraryDatabase(self._database_name)

    def get_new_connection_to_library_database(self):
        library_database = LibraryDatabase(self._database_name)
        if self._database_name == ':memory:':
            # In memory database does not point to the right place..
            # this is here for unit tests..
            library_database.create_database()
        return library_database

    def _handle_message(self):
        message = self._messages.get()
        if not message:
            return False
        msg_type = message[0]
        if msg_type == 'fetch':
            self._handle_fetch_keywords_message(message)
        elif msg_type == 'insert':
            self._handle_insert_keywords_message(message)
        elif msg_type == 'create':
            self._database.create_database()
        return True

    def _handle_fetch_keywords_message(self, message):
        _, library_name, library_args, callback = message
        keywords = self._fetch_keywords(library_name, library_args)
        self._update_database_and_call_callback_if_needed(
            (library_name, library_args), keywords, callback)

    def _fetch_keywords(self, library_name, library_args):
        try:
            path = get_path(
                library_name.replace('/', os.sep), os.path.abspath('.'))
            return get_import_result(path, library_args)
        except Exception as err:
            try:
                print('FAILED', library_name, err)
            except IOError:
                pass
            kws = self._spec_initializer.init_from_spec(library_name)
            if not kws:
                msg = 'Importing test library "%s" failed' % library_name
                RideLogException(
                    message=msg, exception=err, level='WARN').publish()
            return kws

    def _handle_insert_keywords_message(self, message):
        _, library_name, library_args, result_queue = message
        keywords = self._fetch_keywords(library_name, library_args)
        self._insert(library_name, library_args, keywords,
                     lambda res: result_queue.put(res, timeout=3))

    def _insert(self, library_name, library_args, keywords, callback):
        self._database.insert_library_keywords(
            library_name, library_args, keywords or [])
        self._call(callback, keywords)

    def _update_database_and_call_callback_if_needed(
            self, library_key, keywords, callback):
        db_keywords = self._database.fetch_library_keywords(*library_key)
        try:
            if not db_keywords or self._keywords_differ(keywords, db_keywords):
                self._insert(
                    library_key[0], library_key[1], keywords, callback)
            else:
                self._database.update_library_timestamp(*library_key)
        except OperationalError:
            pass

    def _call(self, callback, *args):
        try:
            callback(*args)
        except Exception as err:
            msg = 'Library import callback threw an unexpected exception'
            RideLogException(message=msg, exception=err, level='WARN').publish()

    def fetch_keywords(self, library_name, library_args, callback):
        self._messages.put(('fetch', library_name, library_args, callback),
                           timeout=3)

    def get_and_insert_keywords(self, library_name, library_args):
        result_queue = Queue.Queue(maxsize=1)
        self._messages.put(
            ('insert', library_name, library_args, result_queue), timeout=3)
        try:
            return result_queue.get(timeout=5)
        except Queue.Empty as e:
            RideLogMessage(u'Failed to read keywords from library db: {}'
                           .format(unicode(e))).publish()
            return []

    def create_database(self):
        self._messages.put(('create',), timeout=3)

    def stop(self):
        self._messages.put(False, timeout=3)

    def _keywords_differ(self, keywords1, keywords2):
        if keywords1 != keywords2 and None in (keywords1, keywords2):
            return True
        if len(keywords1) != len(keywords2):
            return True
        for k1, k2 in zip(keywords1, keywords2):
            if k1.name != k2.name:
                return True
            if k1.doc != k2.doc:
                return True
            # if k1.doc_format != k2.doc_format:
            #     return True
            if k1.arguments != k2.arguments:
                return True
            if k1.source != k2.source:
                return True
        return False