示例#1
0
文件: pl_scan.py 项目: maizy/PonyLib
    def handle(self, *args, **options):

        #TODO: use pl_scan_multithread by default. see https://github.com/maizy/PonyLib/issues/48

        self.logger = logging.getLogger('ponylib.scanner')

        lib_paths = map(path.abspath, args)

        stat = Stat()
        stat.start()

        consumer = AddOrUpdateBookConsumer(kwargs={ #multi thread ready consumer
            'connection_alias': 'default',
            'stat': stat})
        consumer.allow_update = False

        iter = Fb2FilesIterator(lib_paths)
        for root_path, rel_path in iter:
            consumer._process_path(root_path, rel_path,
                alias='default', stat=stat)

        stat.end()

        stat.print_report()
        print('\n')
        stat.print_timers()
示例#2
0
    def handle(self, *args, **options):

        self.logger = logging.getLogger('ponylib.scanner')

        #XXX not stable now. Have many gotchas with orm and lxml. up to seg faults.
        #    see: https://github.com/maizy/PonyLib/issues/12#issuecomment-9642515

        #TODO shared connections pool
        connections = self.prepare_and_check_db_connections()

        lib_paths = map(path.abspath, args)

        stat = Stat()
        stat.start()

        pool = ProducerConsumersPool()
        files_queue = Queue.Queue(maxsize=100)

        for i, lib_path in enumerate(lib_paths, start=1):
            producer = Fb2FilesProducer(kwargs={
                'lib_paths' : [lib_path],
                'files_queue': files_queue,
                'stat': stat})
            producer.setName('producer%02d' % i)
            pool.add_producer(producer)

        for i in xrange(1, settings.PONYLIB_SCAN_THREADS+1):
            consumer = AddOrUpdateBookConsumer(kwargs={
                'files_queue': files_queue,
                'connection_alias': connections.pop(),
                'stat': stat})
            consumer.allow_update = False
            consumer.setName('consumer%02d' % i)
            pool.add_consumer(consumer)

        pool.run()

        stat.end()
        stat.print_report()