def _DoParallelOperation(num_threads, tasks, method, label, show_progress_bar):
"""Perform the given storage operation in parallel.
Factors out common work: logging, setting up parallelism, managing a progress
bar (if necessary).
Args:
num_threads: int, the number of threads to use
tasks: list of arguments to be passed to method, one at a time (each zipped
up in a tuple with a callback)
method: a function that takes in a single-argument: a tuple of a task to do
and a zero-argument callback to be done on completion of the task.
label: str, the label for the progress bar (if used).
show_progress_bar: bool, whether to show a progress bar during the
operation.
"""
log.debug(label)
log.debug('Using [%d] threads', num_threads)
pool = parallel.GetPool(num_threads)
if show_progress_bar:
progress_bar = console_io.TickableProgressBar(len(tasks), label)
callback = progress_bar.Tick
else:
progress_bar = console_io.NoOpProgressBar()
callback = None
with progress_bar, pool:
pool.Map(method, list(zip(tasks, itertools.cycle((callback, )))))
def ExecuteTasks(tasks,
num_threads=DEFAULT_NUM_THREADS,
progress_bar_label=None):
"""Perform the given storage tasks in parallel.
Factors out common work: logging, setting up parallelism, managing a progress
bar (if necessary).
Args:
tasks: [Operation], To be executed in parallel.
num_threads: int, The number of threads to use
progress_bar_label: str, If set, a progress bar will be shown with this
label. Otherwise, no progress bar is displayed.
"""
log.debug(progress_bar_label)
log.debug('Using [%d] threads', num_threads)
pool = parallel.GetPool(num_threads)
if progress_bar_label:
progress_bar = console_io.TickableProgressBar(len(tasks),
progress_bar_label)
callback = progress_bar.Tick
else:
progress_bar = console_io.NoOpProgressBar()
callback = None
if num_threads == 0:
with progress_bar:
for t in tasks:
t.Execute(callback)
else:
with progress_bar, pool:
pool.Map(lambda task: task.Execute(callback), tasks)
def testProgressBar(self):
with console_io.TickableProgressBar(4, 'Test Action',
total_ticks=40) as pb:
self.AssertErrNotContains('|')
self.AssertErrContains(
'1========================================2\n'
'3= Test Action =4\n'
'5')
pb.Tick()
self.AssertErrEquals('1========================================2\n'
'3= Test Action =4\n'
'5==========')
pb.Tick()
self.AssertErrEquals('1========================================2\n'
'3= Test Action =4\n'
'5====================')
pb.Tick()
self.AssertErrEquals('1========================================2\n'
'3= Test Action =4\n'
'5==============================')
pb.Tick()
self.AssertErrEquals(
'1========================================2\n'
'3= Test Action =4\n'
'5========================================6\n')
# Should just stay at 100%.
pb.Tick()
self.AssertErrEquals(
'1========================================2\n'
'3= Test Action =4\n'
'5========================================6\n')