def run_sweep_doodad(
target,
params,
run_mode,
mounts,
test_one=False,
docker_image='python:3',
is_docker_interactive=False,
return_output=False,
verbose=False,
postprocess_config_and_run_mode=lambda config, run_mode, idx:
(config, run_mode),
default_params=None):
# build archive
target_dir = os.path.dirname(target)
target_mount_dir = os.path.join('target', os.path.basename(target_dir))
target_mount = mount.MountLocal(local_dir=target_dir,
mount_point=target_mount_dir)
mounts = list(mounts) + [target_mount]
target_full_path = os.path.join(target_mount.mount_point,
os.path.basename(target))
command = launch_api.make_python_command(target_full_path)
print('Launching jobs with mode %s' % run_mode)
results = []
njobs = 0
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(
archive_filename=archive_file,
payload_script=command,
verbose=verbose,
docker_image=docker_image,
is_docker_interactive=is_docker_interactive,
use_nvidia_docker=run_mode.use_gpu,
mounts=mounts)
sweeper = Sweeper(params, default_params)
for config in sweeper:
config, run_mode = postprocess_config_and_run_mode(
config, run_mode, njobs)
if config is None:
continue
njobs += 1
cli_args = ' '.join(
['--%s %s' % (key, config[key]) for key in config])
cmd = archive + ' -- ' + cli_args
result = run_mode.run_script(cmd,
return_output=return_output,
verbose=False)
if return_output:
result = archive_builder._strip_stdout(result)
results.append(result)
if test_one:
break
print('Launching completed for %d jobs' % njobs)
run_mode.print_launch_message()
return tuple(results)
def run_sweep_doodad_chunked(target,
params,
run_mode,
mounts,
num_chunks=10,
docker_image='python:3',
return_output=False,
test_one=False,
confirm=True,
verbose=False):
# build archive
target_dir = os.path.dirname(target)
target_mount_dir = os.path.join('target', os.path.basename(target_dir))
target_mount = mount.MountLocal(local_dir=target_dir,
mount_point=target_mount_dir)
mounts = list(mounts) + [target_mount]
target_full_path = os.path.join(target_mount.mount_point,
os.path.basename(target))
command = launch_api.make_python_command(target_full_path)
print('Launching jobs with mode %s' % run_mode)
results = []
njobs = 0
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(
archive_filename=archive_file,
payload_script=command,
verbose=verbose,
docker_image=docker_image,
use_nvidia_docker=run_mode.use_gpu,
mounts=mounts)
sweeper = Sweeper(params)
chunks = chunker(sweeper, num_chunks, confirm=confirm)
for chunk in chunks:
command = ''
for config in chunk:
njobs += 1
cli_args = ' '.join(
['--%s %s' % (key, config[key]) for key in config])
single_command = archive + ' -- ' + cli_args
command += '%s;' % single_command
result = run_mode.run_script(command,
return_output=return_output,
verbose=False)
if return_output:
result = archive_builder._strip_stdout(result)
results.append(result)
if test_one:
break
print('Launching completed for %d jobs on %d machines' %
(njobs, num_chunks))
run_mode.print_launch_message()
return tuple(results)
def run_command(
command,
cli_args=None,
mode=launch_mode.LocalMode(),
mounts=tuple(),
return_output=False,
verbose=False,
docker_image='ubuntu:18.04',
singularity_image=None,
container_type='docker',
extra_container_flags='',
):
"""
Runs a shell command using doodad via a specified launch mode.
Args:
command (str): A shell command
cli_args (str): Command line args to pass
mode (LaunchMode): A LaunchMode object
mounts (tuple): A list/tuple of Mount objects
return_output (bool): If True, returns stdout as a string.
Do not use if the output will be large.
container_type (string): either 'docker' or 'singularity
extra_container_flags (string): flags other than GPU flags to pass to
docker run or singularity exec.
Returns:
A string output if return_output is True,
else None
"""
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(archive_filename=archive_file,
payload_script=command,
verbose=False,
docker_image=docker_image,
singularity_image=singularity_image,
container_type=container_type,
extra_container_flags=extra_container_flags,
use_gpu_image=mode.use_gpu,
mounts=mounts)
cmd = archive
if cli_args:
cmd = archive + ' -- ' + cli_args
result = mode.run_script(cmd, return_output=return_output, verbose=verbose)
if return_output:
result = archive_builder._strip_stdout(result)
return result
def run_command(command,
cli_args=None,
mode=launch_mode.LocalMode(),
mounts=tuple(),
return_output=False,
verbose=False,
docker_image='ubuntu:18.04'):
"""
Runs a shell command using doodad via a specified launch mode.
Args:
command (str): A shell command
cli_args (str): Command line args to pass
mode (LaunchMode): A LaunchMode object
mounts (tuple): A list/tuple of Mount objects
return_output (bool): If True, returns stdout as a string.
Do not use if the output will be large.
Returns:
A string output if return_output is True,
else None
"""
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(archive_filename=archive_file,
payload_script=command,
verbose=False,
docker_image=docker_image,
use_nvidia_docker=mode.use_gpu,
mounts=mounts)
cmd = archive
if cli_args:
cmd = archive + ' -- ' + cli_args
result = mode.run_script(cmd,
return_output=return_output,
verbose=verbose)
if return_output:
result = archive_builder._strip_stdout(result)
return result