def handle(self, *directories, **options):
        from django.db import transaction

        self.style = color_style()
        
        verbose = options.get('verbose')
        force = options.get('force')
        show_traceback = options.get('traceback', False)
        import_picture = options.get('import_picture')

        wait_until = None
        if options.get('wait_until'):
            wait_until = time.mktime(time.strptime(options.get('wait_until'), '%Y-%m-%d %H:%M:%S'))
            if verbose > 0:
                print "Will wait until %s; it's %f seconds from now" % (
                    time.strftime('%Y-%m-%d %H:%M:%S',
                    time.localtime(wait_until)), wait_until - time.time())

        index = None
        if options.get('search_index') and not settings.NO_SEARCH_INDEX:
            index = Index()
            try:
                index.index_tags()
                index.index.commit()
            except Exception, e:
                index.index.rollback()
                raise e
Beispiel #2
0
    def handle(self, *directories, **options):
        from django.db import transaction

        self.style = color_style()

        verbose = options.get('verbose')
        force = options.get('force')
        show_traceback = options.get('traceback', False)
        import_picture = options.get('import_picture')

        wait_until = None
        if options.get('wait_until'):
            wait_until = time.mktime(
                time.strptime(options.get('wait_until'), '%Y-%m-%d %H:%M:%S'))
            if verbose > 0:
                print "Will wait until %s; it's %f seconds from now" % (
                    time.strftime(
                        '%Y-%m-%d %H:%M:%S',
                        time.localtime(wait_until)), wait_until - time.time())

        index = None
        if options.get('search_index') and not settings.NO_SEARCH_INDEX:
            index = Index()
            try:
                index.index_tags()
                index.index.commit()
            except Exception, e:
                index.index.rollback()
                raise e
    def setUp(self):
        WLTestCase.setUp(self)

        index = Index()
        self.search = Search()
        index.delete_query(self.search.index.query(uid="*"))
        index.index.commit()

        with self.settings(NO_SEARCH_INDEX=False):
            self.do_doktora = Book.from_xml_file(
                get_fixture('do-doktora.xml', opds))
            self.do_anusie = Book.from_xml_file(
                get_fixture('fraszka-do-anusie.xml', catalogue))
Beispiel #4
0
    def setUp(self):
        WLTestCase.setUp(self)

        index = Index()
        self.search = Search()
        index.delete_query(self.search.index.query(uid="*"))
        index.index.commit()

        with self.settings(NO_SEARCH_INDEX=False):
            self.do_doktora = Book.from_xml_file(
                get_fixture('do-doktora.xml', opds))
            self.do_anusie = Book.from_xml_file(
                get_fixture('fraszka-do-anusie.xml', catalogue))
Beispiel #5
0
    def handle(self, *args, **opts):
        index = Index()
        index.open()

        self.delete_old(index)

        try:
            index.optimize()
        finally:
            index.close()
Beispiel #6
0
    def setUp(self):
        JVM.attachCurrentThread()
        WLTestCase.setUp(self)
        settings.NO_SEARCH_INDEX = False
        settings.SEARCH_INDEX = path.join(settings.MEDIA_ROOT, 'search')

        txt = path.join(path.dirname(__file__), 'files/fraszka-do-anusie.xml')
        self.book = models.Book.from_xml_file(txt)

        index = Index()
        index.open()
        try:
            index.index_book(self.book)
        except:
            index.close()

        self.search = Search()
 def setUp(self):
     """
     Setup index that will be subjected to the tests.
     """
     self.index = Index(sample_stop_words())
class IndexTests(unittest.TestCase):
    """
    Test case for Index class.
    """

    def setUp(self):
        """
        Setup index that will be subjected to the tests.
        """
        self.index = Index(sample_stop_words())

    def test_sample_indexing_with_no_exceptions(self):
        sample1 = Indexable(1, "this is an indexable metadata")
        sample2 = Indexable(2, "this is an indexable super metadata")
        sample3 = Indexable(3, "this is another indexable metadata")
        self.index.build_index([sample1, sample2, sample3])

    def test_invalid_term_search(self):
        """
        Test if the search returns when the term is not found.
        """
        sample1 = Indexable(1, "this is an indexable simple metadata")
        sample2 = Indexable(2, "this is an indexable super metadata")
        sample3 = Indexable(3, "this is another indexable metadata")

        expected_indices = []

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(["not_valid_term"])
        self.assertItemsEqual(search_results, expected_indices)

    def test_mixed_valid_invalid_term_search(self):
        """
        Test if the search returns when there are valid and invalid terms mixed.
        """
        sample1 = Indexable(1, "this is an indexable simple metadata")
        sample2 = Indexable(2, "this is an indexable super metadata")
        sample3 = Indexable(3, "this is another indexable metadata")

        expected_indices = []

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(["not_valid_term", "super"])
        self.assertItemsEqual(search_results, expected_indices)

    def test_one_term_search(self):
        """
        Test if the search for one term returns expected results.
        """
        sample1 = Indexable(1, "this is an indexable metadata")
        sample2 = Indexable(2, "this is an indexable super metadata")
        sample3 = Indexable(3, "this is another indexable super metadata")

        expected_indices = [1, 2]

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(["super"])
        self.assertItemsEqual(search_results, expected_indices)

    def test_stop_word_search(self):
        """
        Test if stop words are correctly ignored.
        """
        sample1 = Indexable(1, "this is an indexable metadata")
        sample2 = Indexable(2, "this is an indexable super metadata")
        sample3 = Indexable(3, "this is another indexable super metadata")

        expected_indices = []

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(["this"])
        self.assertItemsEqual(search_results, expected_indices)

    def test_two_terms_search(self):
        """
        Test if the search for two term returns expected results.
        """
        sample1 = Indexable(1, "this is an indexable simple metadata")
        sample2 = Indexable(2, "this is an indexable super metadata")
        sample3 = Indexable(3, "this is another indexable super metadata")

        expected_indices = [1, 2]

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(["indexable", "super"])
        self.assertItemsEqual(search_results, expected_indices)
Beispiel #9
0
    def handle(self, *directories, **options):
        from django.db import transaction

        self.style = color_style()

        verbose = options.get('verbose')
        force = options.get('force')
        show_traceback = options.get('traceback', False)
        import_picture = options.get('import_picture')

        wait_until = None
        if options.get('wait_until'):
            wait_until = time.mktime(time.strptime(options.get('wait_until'), '%Y-%m-%d %H:%M:%S'))
            if verbose > 0:
                print "Will wait until %s; it's %f seconds from now" % (
                    time.strftime('%Y-%m-%d %H:%M:%S',
                    time.localtime(wait_until)), wait_until - time.time())

        if options.get('search_index') and not settings.NO_SEARCH_INDEX:
            index = Index()
            index.open()
            try:
                index.index_tags()
            finally:
                index.close()

        # Start transaction management.
        transaction.commit_unless_managed()
        transaction.enter_transaction_management()
        transaction.managed(True)

        files_imported = 0
        files_skipped = 0
        
        for dir_name in directories:
            if not os.path.isdir(dir_name):
                print self.style.ERROR("%s: Not a directory. Skipping." % dir_name)
            else:
                # files queue
                files = sorted(os.listdir(dir_name))
                postponed = {}
                while files:
                    file_name = files.pop(0)
                    file_path = os.path.join(dir_name, file_name)
                    file_base, ext = os.path.splitext(file_path)

                    # Skip files that are not XML files
                    if not ext == '.xml':
                        continue

                    if verbose > 0:
                        print "Parsing '%s'" % file_path
                    else:
                        sys.stdout.write('.')
                        sys.stdout.flush()

                    # Import book files
                    try:
                        if import_picture:
                            self.import_picture(file_path, options)
                        else:
                            self.import_book(file_path, options)
                        files_imported += 1
                        transaction.commit()
                        
                    except (Book.AlreadyExists, Picture.AlreadyExists):
                        print self.style.ERROR('%s: Book or Picture already imported. Skipping. To overwrite use --force.' %
                            file_path)
                        files_skipped += 1

                    except Book.DoesNotExist, e:
                        if file_name not in postponed or postponed[file_name] < files_imported:
                            # push it back into the queue, maybe the missing child will show up
                            if verbose:
                                print self.style.NOTICE('Waiting for missing children')
                            files.append(file_name)
                            postponed[file_name] = files_imported
                        else:
                            # we're in a loop, nothing's being imported - some child is really missing
                            raise e
 def setUp(self):
     """
     Setup index that will be subjected to the tests.
     """
     self.index = Index(sample_stop_words())
class IndexTests(unittest.TestCase):
    """
    Test case for Index class.
    """
    def setUp(self):
        """
        Setup index that will be subjected to the tests.
        """
        self.index = Index(sample_stop_words())

    def test_sample_indexing_with_no_exceptions(self):
        sample1 = Indexable(1, 'this is an indexable metadata')
        sample2 = Indexable(2, 'this is an indexable super metadata')
        sample3 = Indexable(3, 'this is another indexable metadata')
        self.index.build_index([sample1, sample2, sample3])

    def test_invalid_term_search(self):
        """
        Test if the search returns when the term is not found.
        """
        sample1 = Indexable(1, 'this is an indexable simple metadata')
        sample2 = Indexable(2, 'this is an indexable super metadata')
        sample3 = Indexable(3, 'this is another indexable metadata')

        expected_indices = []

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(['not_valid_term'])
        self.assertItemsEqual(search_results, expected_indices)

    def test_mixed_valid_invalid_term_search(self):
        """
        Test if the search returns when there are valid and invalid terms mixed.
        """
        sample1 = Indexable(1, 'this is an indexable simple metadata')
        sample2 = Indexable(2, 'this is an indexable super metadata')
        sample3 = Indexable(3, 'this is another indexable metadata')

        expected_indices = []

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(['not_valid_term', 'super'])
        self.assertItemsEqual(search_results, expected_indices)

    def test_one_term_search(self):
        """
        Test if the search for one term returns expected results.
        """
        sample1 = Indexable(1, 'this is an indexable metadata')
        sample2 = Indexable(2, 'this is an indexable super metadata')
        sample3 = Indexable(3, 'this is another indexable super metadata')

        expected_indices = [1, 2]

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(['super'])
        self.assertItemsEqual(search_results, expected_indices)

    def test_stop_word_search(self):
        """
        Test if stop words are correctly ignored.
        """
        sample1 = Indexable(1, 'this is an indexable metadata')
        sample2 = Indexable(2, 'this is an indexable super metadata')
        sample3 = Indexable(3, 'this is another indexable super metadata')

        expected_indices = []

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(['this'])
        self.assertItemsEqual(search_results, expected_indices)

    def test_two_terms_search(self):
        """
        Test if the search for two term returns expected results.
        """
        sample1 = Indexable(1, 'this is an indexable simple metadata')
        sample2 = Indexable(2, 'this is an indexable super metadata')
        sample3 = Indexable(3, 'this is another indexable super metadata')

        expected_indices = [1, 2]

        self.index.build_index([sample1, sample2, sample3])
        search_results = self.index.search_terms(['indexable', 'super'])
        self.assertItemsEqual(search_results, expected_indices)
Beispiel #12
0
    }, {
        'selector': '3',
        'prompt': 'Exit',
        'return': 'exit'
    }]
    selection = prompt.options("Browser or Terminal Engine", options)

    if selection == 'exit':
        print 'Bye'
        exit()

    global my_index, dirname
    dirname = path = prompt.query('Documents Path',
                                  default=os.getcwd(),
                                  validators=[validators.PathValidator()])
    my_index = Index(dirname)

    if selection == 'web':
        app.run(debug=False)
    elif selection == 'cmd':
        while 1:
            options = [{
                'selector': '1',
                'prompt': 'Query',
                'return': 'cont'
            }, {
                'selector': '2',
                'prompt': 'Exit',
                'return': 'exit'
            }]