def test_build_parallel_subprocess(user_modules, dispatcher_options):
parallel_mode = 'subprocess'
parallel = build_parallel(
parallel_mode=parallel_mode,
user_modules=user_modules,
dispatcher_options=dispatcher_options
)
## progressMonitor
assert 'NullProgressMonitor' == parallel.progressMonitor.__class__.__name__
## communicationChannel
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'TaskPackageDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## dispatcher
assert 'SubprocessRunner' == parallel.communicationChannel.dropbox.dispatcher.__class__.__name__
assert 'WorkingArea' == parallel.workingarea.__class__.__name__
if user_modules:
assert set(['scribblers', 'alphatwirl']) == set(parallel.workingarea.python_modules)
else:
assert set(['alphatwirl']) == set(parallel.workingarea.python_modules)
def test_build_parallel_multiprocessing(quiet, processes, isatty):
parallel_mode = 'multiprocessing'
parallel = build_parallel(
parallel_mode=parallel_mode,
quiet=quiet,
processes=processes,
)
## progressMonitor
if quiet:
assert 'NullProgressMonitor' == parallel.progressMonitor.__class__.__name__
elif processes == 0:
assert 'ProgressMonitor' == parallel.progressMonitor.__class__.__name__
else:
assert 'BProgressMonitor' == parallel.progressMonitor.__class__.__name__
if not quiet:
if isatty:
assert 'ProgressBar' == parallel.progressMonitor.presentation.__class__.__name__
else:
assert 'ProgressPrint' == parallel.progressMonitor.presentation.__class__.__name__
## communicationChannel
if processes == 0:
assert 'CommunicationChannel0' == parallel.communicationChannel.__class__.__name__
else:
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'MultiprocessingDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## workingarea
assert parallel.workingarea is None
def test_build_parallel_subprocess(user_modules, dispatcher_options):
parallel_mode = 'subprocess'
parallel = build_parallel(parallel_mode=parallel_mode,
user_modules=user_modules,
dispatcher_options=dispatcher_options)
## progressMonitor
assert 'NullProgressMonitor' == parallel.progressMonitor.__class__.__name__
## communicationChannel
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'TaskPackageDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## dispatcher
assert 'SubprocessRunner' == parallel.communicationChannel.dropbox.dispatcher.__class__.__name__
assert 'WorkingArea' == parallel.workingarea.__class__.__name__
if user_modules:
assert set(['scribblers',
'alphatwirl']) == set(parallel.workingarea.python_modules)
else:
assert set(['alphatwirl']) == set(parallel.workingarea.python_modules)
def test_build_parallel_htcondor(
user_modules, htcondor_job_desc_extra,
dispatcher_options):
parallel_mode = 'htcondor'
parallel = build_parallel(
parallel_mode=parallel_mode,
user_modules=user_modules,
htcondor_job_desc_extra=htcondor_job_desc_extra,
dispatcher_options=dispatcher_options
)
## progressMonitor
assert 'NullProgressMonitor' == parallel.progressMonitor.__class__.__name__
## communicationChannel
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'TaskPackageDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## dispatcher
assert 'HTCondorJobSubmitter' == parallel.communicationChannel.dropbox.dispatcher.__class__.__name__
assert htcondor_job_desc_extra == parallel.communicationChannel.dropbox.dispatcher.job_desc_extra
if 'job_desc_dict' in dispatcher_options:
assert dispatcher_options['job_desc_dict'] == parallel.communicationChannel.dropbox.dispatcher.user_job_desc_dict
assert 'WorkingArea' == parallel.workingarea.__class__.__name__
if user_modules:
assert set(['scribblers', 'alphatwirl']) == set(parallel.workingarea.python_modules)
else:
assert set(['alphatwirl']) == set(parallel.workingarea.python_modules)
def test_build_parallel_multiprocessing(quiet, processes, isatty):
parallel_mode = 'multiprocessing'
parallel = build_parallel(
parallel_mode=parallel_mode,
quiet=quiet,
processes=processes,
)
## progressMonitor
if quiet:
assert 'NullProgressMonitor' == parallel.progressMonitor.__class__.__name__
elif processes == 0:
assert 'ProgressMonitor' == parallel.progressMonitor.__class__.__name__
else:
assert 'BProgressMonitor' == parallel.progressMonitor.__class__.__name__
if not quiet:
if isatty:
assert 'ProgressBar' == parallel.progressMonitor.presentation.__class__.__name__
else:
assert 'ProgressPrint' == parallel.progressMonitor.presentation.__class__.__name__
## communicationChannel
if processes == 0:
assert 'CommunicationChannel0' == parallel.communicationChannel.__class__.__name__
else:
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'MultiprocessingDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## workingarea
assert parallel.workingarea is None
def test_build_parallel_htcondor(user_modules, htcondor_job_desc_extra,
dispatcher_options):
parallel_mode = 'htcondor'
parallel = build_parallel(parallel_mode=parallel_mode,
user_modules=user_modules,
htcondor_job_desc_extra=htcondor_job_desc_extra,
dispatcher_options=dispatcher_options)
## progressMonitor
assert 'NullProgressMonitor' == parallel.progressMonitor.__class__.__name__
## communicationChannel
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'TaskPackageDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## dispatcher
assert 'HTCondorJobSubmitter' == parallel.communicationChannel.dropbox.dispatcher.__class__.__name__
assert htcondor_job_desc_extra == parallel.communicationChannel.dropbox.dispatcher.job_desc_extra
if 'job_desc_dict' in dispatcher_options:
assert dispatcher_options[
'job_desc_dict'] == parallel.communicationChannel.dropbox.dispatcher.user_job_desc_dict
assert 'WorkingArea' == parallel.workingarea.__class__.__name__
if user_modules:
assert set(['scribblers',
'alphatwirl']) == set(parallel.workingarea.python_modules)
else:
assert set(['alphatwirl']) == set(parallel.workingarea.python_modules)
def __init__(self,
outdir,
force=False,
quiet=False,
parallel_mode='multiprocessing',
dispatcher_options=dict(),
process=4,
user_modules=(),
max_blocks_per_dataset=-1,
max_blocks_per_process=-1,
max_files_per_dataset=-1,
max_files_per_process=1,
nevents_per_block=1000000,
profile=False,
profile_out_path=None):
self.parallel = build_parallel(parallel_mode=parallel_mode,
quiet=quiet,
processes=process,
user_modules=user_modules,
dispatcher_options=dispatcher_options)
self.outdir = outdir
self.force = force
self.max_blocks_per_dataset = max_blocks_per_dataset
self.max_blocks_per_process = max_blocks_per_process
self.max_files_per_dataset = max_files_per_dataset
self.max_files_per_process = max_files_per_process
self.nevents_per_block = nevents_per_block
self.profile = profile
self.profile_out_path = profile_out_path
self.parallel_mode = parallel_mode
def test_build_parallel_htcondor(user_modules, dispatcher_options,
wrap_HTCondorJobSubmitter):
parallel_mode = 'htcondor'
parallel = build_parallel(parallel_mode=parallel_mode,
user_modules=user_modules,
dispatcher_options=dispatcher_options)
## communicationChannel
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'TaskPackageDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
## dispatcher
assert 'HTCondorJobSubmitter' == parallel.communicationChannel.dropbox.dispatcher.__class__.__name__
if 'job_desc_dict' in dispatcher_options:
assert [mock.call(job_desc_dict={'request_memory': '120'})
] == wrap_HTCondorJobSubmitter.call_args_list
assert 'WorkingArea' == parallel.workingarea.__class__.__name__
if user_modules:
assert set(['scribblers', 'alphatwirl', 'atpbar',
'mantichora']) == set(parallel.workingarea.python_modules)
else:
assert set(['alphatwirl', 'atpbar',
'mantichora']) == set(parallel.workingarea.python_modules)
def test_build_logging_unknown_parallel_mode(caplog):
with caplog.at_level(logging.WARNING):
parallel = build_parallel(parallel_mode='unknown_mode')
assert len(caplog.records) == 1
assert caplog.records[0].levelname == 'WARNING'
assert 'parallel.build' in caplog.records[0].name
assert 'unknown parallel_mode' in caplog.records[0].msg
assert 'MultiprocessingDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
def test_build_logging_unknown_parallel_mode(caplog):
with caplog.at_level(logging.WARNING):
parallel = build_parallel(parallel_mode='unknown_mode')
assert len(caplog.records) == 1
assert caplog.records[0].levelname == 'WARNING'
assert 'parallel.build' in caplog.records[0].name
assert 'unknown parallel_mode' in caplog.records[0].msg
assert 'MultiprocessingDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
def test_removed_job_desc_extra(caplog):
parallel_mode = 'htcondor'
htcondor_job_desc_extra = ['request_memory = 250']
with pytest.raises(TypeError):
with caplog.at_level(logging.ERROR):
parallel = build_parallel(
parallel_mode=parallel_mode,
htcondor_job_desc_extra=htcondor_job_desc_extra)
assert len(caplog.records) == 1
assert caplog.records[0].levelname == 'ERROR'
assert 'parallel.build' in caplog.records[0].name
assert '"htcondor_job_desc_extra" is removed.' in caplog.records[0].msg
def test_build_parallel_multiprocessing(quiet, processes, mock_atpbar):
parallel_mode = 'multiprocessing'
parallel = build_parallel(
parallel_mode=parallel_mode,
quiet=quiet,
processes=processes,
)
## communicationChannel
if processes == 0:
assert 'CommunicationChannel0' == parallel.communicationChannel.__class__.__name__
else:
assert 'CommunicationChannel' == parallel.communicationChannel.__class__.__name__
assert 'MultiprocessingDropbox' == parallel.communicationChannel.dropbox.__class__.__name__
if quiet:
assert [mock.call()] == mock_atpbar.disable.call_args_list
else:
assert [] == mock_atpbar.disable.call_args_list
## workingarea
assert parallel.workingarea is None