def _DoParallelOperation(num_threads, tasks, method, label, show_progress_bar): """Perform the given storage operation in parallel. Factors out common work: logging, setting up parallelism, managing a progress bar (if necessary). Args: num_threads: int, the number of threads to use tasks: list of arguments to be passed to method, one at a time (each zipped up in a tuple with a callback) method: a function that takes in a single-argument: a tuple of a task to do and a zero-argument callback to be done on completion of the task. label: str, the label for the progress bar (if used). show_progress_bar: bool, whether to show a progress bar during the operation. """ log.debug(label) log.debug('Using [%d] threads', num_threads) pool = parallel.GetPool(num_threads) if show_progress_bar: progress_bar = console_io.TickableProgressBar(len(tasks), label) callback = progress_bar.Tick else: progress_bar = console_io.NoOpProgressBar() callback = None with progress_bar, pool: pool.Map(method, list(zip(tasks, itertools.cycle((callback, )))))
def ExecuteTasks(tasks, num_threads=DEFAULT_NUM_THREADS, progress_bar_label=None): """Perform the given storage tasks in parallel. Factors out common work: logging, setting up parallelism, managing a progress bar (if necessary). Args: tasks: [Operation], To be executed in parallel. num_threads: int, The number of threads to use progress_bar_label: str, If set, a progress bar will be shown with this label. Otherwise, no progress bar is displayed. """ log.debug(progress_bar_label) log.debug('Using [%d] threads', num_threads) pool = parallel.GetPool(num_threads) if progress_bar_label: progress_bar = console_io.TickableProgressBar(len(tasks), progress_bar_label) callback = progress_bar.Tick else: progress_bar = console_io.NoOpProgressBar() callback = None if num_threads == 0: with progress_bar: for t in tasks: t.Execute(callback) else: with progress_bar, pool: pool.Map(lambda task: task.Execute(callback), tasks)
def testProgressBar(self): with console_io.TickableProgressBar(4, 'Test Action', total_ticks=40) as pb: self.AssertErrNotContains('|') self.AssertErrContains( '1========================================2\n' '3= Test Action =4\n' '5') pb.Tick() self.AssertErrEquals('1========================================2\n' '3= Test Action =4\n' '5==========') pb.Tick() self.AssertErrEquals('1========================================2\n' '3= Test Action =4\n' '5====================') pb.Tick() self.AssertErrEquals('1========================================2\n' '3= Test Action =4\n' '5==============================') pb.Tick() self.AssertErrEquals( '1========================================2\n' '3= Test Action =4\n' '5========================================6\n') # Should just stay at 100%. pb.Tick() self.AssertErrEquals( '1========================================2\n' '3= Test Action =4\n' '5========================================6\n')