예제 #1
0
파일: pl_scan.py 프로젝트: maizy/PonyLib
    def handle(self, *args, **options):

        #TODO: use pl_scan_multithread by default. see https://github.com/maizy/PonyLib/issues/48

        self.logger = logging.getLogger('ponylib.scanner')

        lib_paths = map(path.abspath, args)

        stat = Stat()
        stat.start()

        consumer = AddOrUpdateBookConsumer(kwargs={ #multi thread ready consumer
            'connection_alias': 'default',
            'stat': stat})
        consumer.allow_update = False

        iter = Fb2FilesIterator(lib_paths)
        for root_path, rel_path in iter:
            consumer._process_path(root_path, rel_path,
                alias='default', stat=stat)

        stat.end()

        stat.print_report()
        print('\n')
        stat.print_timers()
예제 #2
0
    def handle(self, *args, **options):

        self.logger = logging.getLogger('ponylib.scanner')

        #XXX not stable now. Have many gotchas with orm and lxml. up to seg faults.
        #    see: https://github.com/maizy/PonyLib/issues/12#issuecomment-9642515

        #TODO shared connections pool
        connections = self.prepare_and_check_db_connections()

        lib_paths = map(path.abspath, args)

        stat = Stat()
        stat.start()

        pool = ProducerConsumersPool()
        files_queue = Queue.Queue(maxsize=100)

        for i, lib_path in enumerate(lib_paths, start=1):
            producer = Fb2FilesProducer(kwargs={
                'lib_paths' : [lib_path],
                'files_queue': files_queue,
                'stat': stat})
            producer.setName('producer%02d' % i)
            pool.add_producer(producer)

        for i in xrange(1, settings.PONYLIB_SCAN_THREADS+1):
            consumer = AddOrUpdateBookConsumer(kwargs={
                'files_queue': files_queue,
                'connection_alias': connections.pop(),
                'stat': stat})
            consumer.allow_update = False
            consumer.setName('consumer%02d' % i)
            pool.add_consumer(consumer)

        pool.run()

        stat.end()
        stat.print_report()
예제 #3
0
파일: scanner.py 프로젝트: maizy/PonyLib
    def test_over_max_length_strings(self):
        """Issue #52"""

        series_index_maxlen = BookSeries._meta.get_field_by_name('number')[0].max_length
        genre_code_maxlen = Genre._meta.get_field_by_name('code')[0].max_length
        #format in ponylib.meta.read_fb2_meta()
        mock_meta = {
            #unlimited field
            'title': generate_rand_word(1986),

            #limited fields
            'genres': [generate_rand_word(genre_code_maxlen*2)],
            'series': [{'name': generate_rand_word(1111), 'index': generate_rand_word(series_index_maxlen*2)}]
        }

        with patch.object(AddOrUpdateBookConsumer, '_read_meta_data', return_value=(mock_meta, True)):

            consumer = AddOrUpdateBookConsumer(kwargs={'connection_alias': 'default'})
            consumer.allow_update = False

            book = consumer._process_path(self.sample_root, self.sample_fb2)
            book_id = book.id
            book_from_db = Book.objects.get(pk=book_id)

            #unlimited field
            self.assertEqual(book_from_db.title, mock_meta['title'])

            #limited fiels
            self.assertEqual(book_from_db.series.count(), 1)
            self.assertEquals(
                book_from_db.get_series_links()[0].number,
                mock_meta['series'][0]['index'][:series_index_maxlen-3] + '...')

            self.assertEqual(book_from_db.genres.count(), 1)
            self.assertEquals(book_from_db.genres.all()[0].code, mock_meta['genres'][0][:genre_code_maxlen])


    #TODO: consumer_without_stat_test