def _prepare_worker_pools(self):
'''
@ParseDirectiveMixin._prepare_worker_pools
'''
if self.pools is None:
self.pools = Container()
self.pools.progress = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
None,
daemonize=False,
worker_class=parallel.ProgressTrackerWorker,
worker_count=1,
worker_kwargs=dict(\
pcount=len(self.frontier),
pdesc='Total',
punit='files'\
)\
)
self.pools.parser = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
self._TASK_CLASS,
daemonize=False,
worker_count=self.args.threads,
worker_kwargs=self._get_worker_kwargs(),
task_kwargs=self._get_task_kwargs()\
)
def _prepare_worker_pools(self):
'''
@ParseDirectiveMixin._prepare_worker_pools
'''
if self.pools is None:
self.pools = Container()
self.pools.progress = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage2,
daemonize=False,
worker_class=parallel.DBProgressTrackerWorker,
worker_count=1,
worker_kwargs=dict(\
log_path=self.args.log_path,
pcount=len(self.frontier),
pdesc='Total',
punit='files',
manager=DBManager(conn_string=self.conn_string)\
)\
)
self.pools.parser = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage1,
daemonize=False,
worker_count=self.args.threads,
worker_kwargs=dict(\
result_queue=self.pools.progress.queue,
log_path=self.args.log_path\
)
)
def _prepare_worker_pools(self):
'''
@ParseDirectiveMixin._prepare_worker_pools
'''
if self.pools is None:
self.pools = Container()
self.pools.progress = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage2,
daemonize=False,
worker_class=parallel.DBProgressTrackerWorker,
worker_count=1\
)
self.pools.parser = parallel.WorkerPool(\
parallel.JoinableQueue(-1),
tasks.ParseDBTaskStage1,
daemonize=False,
worker_count=self.args.threads,
worker_kwargs=dict(\
result_queue=self.pools.progress.queue,
log_path=self.args.log_path\
)
)