def handle(self, *args, **options):
#TODO: use pl_scan_multithread by default. see https://github.com/maizy/PonyLib/issues/48
self.logger = logging.getLogger('ponylib.scanner')
lib_paths = map(path.abspath, args)
stat = Stat()
stat.start()
consumer = AddOrUpdateBookConsumer(kwargs={ #multi thread ready consumer
'connection_alias': 'default',
'stat': stat})
consumer.allow_update = False
iter = Fb2FilesIterator(lib_paths)
for root_path, rel_path in iter:
consumer._process_path(root_path, rel_path,
alias='default', stat=stat)
stat.end()
stat.print_report()
print('\n')
stat.print_timers()
def handle(self, *args, **options):
self.logger = logging.getLogger('ponylib.scanner')
#XXX not stable now. Have many gotchas with orm and lxml. up to seg faults.
# see: https://github.com/maizy/PonyLib/issues/12#issuecomment-9642515
#TODO shared connections pool
connections = self.prepare_and_check_db_connections()
lib_paths = map(path.abspath, args)
stat = Stat()
stat.start()
pool = ProducerConsumersPool()
files_queue = Queue.Queue(maxsize=100)
for i, lib_path in enumerate(lib_paths, start=1):
producer = Fb2FilesProducer(kwargs={
'lib_paths' : [lib_path],
'files_queue': files_queue,
'stat': stat})
producer.setName('producer%02d' % i)
pool.add_producer(producer)
for i in xrange(1, settings.PONYLIB_SCAN_THREADS+1):
consumer = AddOrUpdateBookConsumer(kwargs={
'files_queue': files_queue,
'connection_alias': connections.pop(),
'stat': stat})
consumer.allow_update = False
consumer.setName('consumer%02d' % i)
pool.add_consumer(consumer)
pool.run()
stat.end()
stat.print_report()