def test_nested_loops(mock_create_presentation, wrap_end_pickup):
disable()
for i in atpbar(range(4)):
for j in atpbar(range(3)):
pass
## print()
## print(mock_create_presentation)
assert 0 == wrap_end_pickup.call_count
def build_parallel(parallel_mode,
quiet=True,
processes=4,
user_modules=None,
dispatcher_options=None):
"""initializes `Parallel`
Parameters
----------
parallel_mode : str
"multiprocessing" (default), "htcondor" or "subprocess"
quiet : bool, optional
if True, progress bars will not be shown in the "multiprocessing" mode.
process : int, optional
The number of processes when ``parallel_mode`` is
"multiprocessing"
user_modules : list, optional
The names of modules to be sent to worker nodes when
parallel_mode is "htcondor"
dispatcher_options : dict, optional
Options to dispatcher
Returns
-------
parallel
an instance of the class `Parallel`
"""
if user_modules is None:
user_modules = []
if dispatcher_options is None:
dispatcher_options = dict()
dispatchers = ('subprocess', 'htcondor')
parallel_modes = ('multiprocessing', ) + dispatchers
default_parallel_mode = 'multiprocessing'
if not parallel_mode in parallel_modes:
logger = logging.getLogger(__name__)
logger.warning('unknown parallel_mode "{}", use default "{}"'.format(
parallel_mode, default_parallel_mode))
parallel_mode = default_parallel_mode
if parallel_mode == 'multiprocessing':
if quiet:
atpbar.disable()
return _build_parallel_multiprocessing(processes=processes)
return _build_parallel_dropbox(parallel_mode=parallel_mode,
user_modules=user_modules,
dispatcher_options=dispatcher_options)
def test_threading(mock_create_presentation, nthreads, niterations):
disable()
# make niterations as long as nthreads. repeat if necessary
niterations = list(
itertools.chain(
*itertools.repeat(niterations, nthreads // len(niterations) +
1)))[:nthreads]
run_with_threading(nthreads, niterations)
presentations = mock_create_presentation.presentations
assert 0 == len(presentations)
def test_multiprocessing(mock_create_presentation, nprocesses, ntasks,
niterations):
disable()
# make niterations as long as ntasks. repeat if necessary
niterations = list(
itertools.chain(
*itertools.repeat(niterations, ntasks // len(niterations) +
1)))[:ntasks]
run_with_multiprocessing(nprocesses, ntasks, niterations)
presentations = mock_create_presentation.presentations
assert 0 == len(presentations)
def test_one_loop(mock_create_presentation, wrap_end_pickup, niterations):
#
disable()
#
for i in atpbar(range(niterations)):
pass
print()
print(mock_create_presentation)
#
assert 0 == wrap_end_pickup.call_count
#
presentations = mock_create_presentation.presentations
assert 0 == len(presentations)
def test_threading(mock_create_presentation, wrap_end_pickup, nthreads,
niterations):
disable()
# make niterations as long as nthreads. repeat if necessary
niterations = list(
itertools.chain(
*itertools.repeat(niterations, nthreads // len(niterations) +
1)))[:nthreads]
run_with_threading(nthreads, niterations)
## print()
## print(mock_create_presentation)
assert 1 == wrap_end_pickup.call_count
presentations = mock_create_presentation.presentations
assert 0 == len(presentations)
def test_nested_loops(mock_create_presentation):
disable()
for i in atpbar(range(4)):
for j in atpbar(range(3)):
pass
def test_one_loop(mock_create_presentation, niterations):
disable()
for i in atpbar(range(niterations)):
pass
presentations = mock_create_presentation.presentations
assert 0 == len(presentations)
def test_call_twice():
disable()
disable()
from tqdm.auto import tqdm
from alphatwirl.datasetloop import CollectorComposite
from atpbar import disable
disable()
class CustomCollectorComposite(CollectorComposite):
def collect(self, dataset_readers_list):
ret = []
for i, collector in enumerate(
tqdm(
self.components,
desc="collecting results",
)):
ret.append(
collector.collect([
(dataset,
tuple(r.readers[i] for r in readerComposites
if hasattr(r, "readers")))
for dataset, readerComposites in dataset_readers_list
]))
return ret
def disable_progressbar():
try:
import atpbar
atpbar.disable()
except:
pass