def handle(self, *args, **options):
#TODO: use pl_scan_multithread by default. see https://github.com/maizy/PonyLib/issues/48
self.logger = logging.getLogger('ponylib.scanner')
lib_paths = map(path.abspath, args)
stat = Stat()
stat.start()
consumer = AddOrUpdateBookConsumer(kwargs={ #multi thread ready consumer
'connection_alias': 'default',
'stat': stat})
consumer.allow_update = False
iter = Fb2FilesIterator(lib_paths)
for root_path, rel_path in iter:
consumer._process_path(root_path, rel_path,
alias='default', stat=stat)
stat.end()
stat.print_report()
print('\n')
stat.print_timers()
def handle(self, *args, **options):
self.logger = logging.getLogger('ponylib.scanner')
#XXX not stable now. Have many gotchas with orm and lxml. up to seg faults.
# see: https://github.com/maizy/PonyLib/issues/12#issuecomment-9642515
#TODO shared connections pool
connections = self.prepare_and_check_db_connections()
lib_paths = map(path.abspath, args)
stat = Stat()
stat.start()
pool = ProducerConsumersPool()
files_queue = Queue.Queue(maxsize=100)
for i, lib_path in enumerate(lib_paths, start=1):
producer = Fb2FilesProducer(kwargs={
'lib_paths' : [lib_path],
'files_queue': files_queue,
'stat': stat})
producer.setName('producer%02d' % i)
pool.add_producer(producer)
for i in xrange(1, settings.PONYLIB_SCAN_THREADS+1):
consumer = AddOrUpdateBookConsumer(kwargs={
'files_queue': files_queue,
'connection_alias': connections.pop(),
'stat': stat})
consumer.allow_update = False
consumer.setName('consumer%02d' % i)
pool.add_consumer(consumer)
pool.run()
stat.end()
stat.print_report()
def test_over_max_length_strings(self):
"""Issue #52"""
series_index_maxlen = BookSeries._meta.get_field_by_name('number')[0].max_length
genre_code_maxlen = Genre._meta.get_field_by_name('code')[0].max_length
#format in ponylib.meta.read_fb2_meta()
mock_meta = {
#unlimited field
'title': generate_rand_word(1986),
#limited fields
'genres': [generate_rand_word(genre_code_maxlen*2)],
'series': [{'name': generate_rand_word(1111), 'index': generate_rand_word(series_index_maxlen*2)}]
}
with patch.object(AddOrUpdateBookConsumer, '_read_meta_data', return_value=(mock_meta, True)):
consumer = AddOrUpdateBookConsumer(kwargs={'connection_alias': 'default'})
consumer.allow_update = False
book = consumer._process_path(self.sample_root, self.sample_fb2)
book_id = book.id
book_from_db = Book.objects.get(pk=book_id)
#unlimited field
self.assertEqual(book_from_db.title, mock_meta['title'])
#limited fiels
self.assertEqual(book_from_db.series.count(), 1)
self.assertEquals(
book_from_db.get_series_links()[0].number,
mock_meta['series'][0]['index'][:series_index_maxlen-3] + '...')
self.assertEqual(book_from_db.genres.count(), 1)
self.assertEquals(book_from_db.genres.all()[0].code, mock_meta['genres'][0][:genre_code_maxlen])
#TODO: consumer_without_stat_test