def test_verbose(self):
payload_script = 'echo hello123'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=True,
docker_image='python:3')
output, errors = archive_builder_docker.run_archive(archive, timeout=5)
output = output.strip()
self.assertEqual(output, 'hello123')
def run_sweep_doodad(
target,
params,
run_mode,
mounts,
test_one=False,
docker_image='python:3',
is_docker_interactive=False,
return_output=False,
verbose=False,
postprocess_config_and_run_mode=lambda config, run_mode, idx:
(config, run_mode),
default_params=None):
# build archive
target_dir = os.path.dirname(target)
target_mount_dir = os.path.join('target', os.path.basename(target_dir))
target_mount = mount.MountLocal(local_dir=target_dir,
mount_point=target_mount_dir)
mounts = list(mounts) + [target_mount]
target_full_path = os.path.join(target_mount.mount_point,
os.path.basename(target))
command = launch_api.make_python_command(target_full_path)
print('Launching jobs with mode %s' % run_mode)
results = []
njobs = 0
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(
archive_filename=archive_file,
payload_script=command,
verbose=verbose,
docker_image=docker_image,
is_docker_interactive=is_docker_interactive,
use_nvidia_docker=run_mode.use_gpu,
mounts=mounts)
sweeper = Sweeper(params, default_params)
for config in sweeper:
config, run_mode = postprocess_config_and_run_mode(
config, run_mode, njobs)
if config is None:
continue
njobs += 1
cli_args = ' '.join(
['--%s %s' % (key, config[key]) for key in config])
cmd = archive + ' -- ' + cli_args
result = run_mode.run_script(cmd,
return_output=return_output,
verbose=False)
if return_output:
result = archive_builder._strip_stdout(result)
results.append(result)
if test_one:
break
print('Launching completed for %d jobs' % njobs)
run_mode.print_launch_message()
return tuple(results)
def run_sweep_doodad_chunked(target,
params,
run_mode,
mounts,
num_chunks=10,
docker_image='python:3',
return_output=False,
test_one=False,
confirm=True,
verbose=False):
# build archive
target_dir = os.path.dirname(target)
target_mount_dir = os.path.join('target', os.path.basename(target_dir))
target_mount = mount.MountLocal(local_dir=target_dir,
mount_point=target_mount_dir)
mounts = list(mounts) + [target_mount]
target_full_path = os.path.join(target_mount.mount_point,
os.path.basename(target))
command = launch_api.make_python_command(target_full_path)
print('Launching jobs with mode %s' % run_mode)
results = []
njobs = 0
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(
archive_filename=archive_file,
payload_script=command,
verbose=verbose,
docker_image=docker_image,
use_nvidia_docker=run_mode.use_gpu,
mounts=mounts)
sweeper = Sweeper(params)
chunks = chunker(sweeper, num_chunks, confirm=confirm)
for chunk in chunks:
command = ''
for config in chunk:
njobs += 1
cli_args = ' '.join(
['--%s %s' % (key, config[key]) for key in config])
single_command = archive + ' -- ' + cli_args
command += '%s;' % single_command
result = run_mode.run_script(command,
return_output=return_output,
verbose=False)
if return_output:
result = archive_builder._strip_stdout(result)
results.append(result)
if test_one:
break
print('Launching completed for %d jobs on %d machines' %
(njobs, num_chunks))
run_mode.print_launch_message()
return tuple(results)
def test_cli_args(self):
payload_script = 'echo hi'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=True,
docker_image='python:3')
output, errors = archive_builder_docker.run_archive(archive,
cli_args='--help',
timeout=5)
output = output.strip()
self.assertEqual(output, 'hi --help')
def run_command(
command,
cli_args=None,
mode=launch_mode.LocalMode(),
mounts=tuple(),
return_output=False,
verbose=False,
docker_image='ubuntu:18.04',
singularity_image=None,
container_type='docker',
extra_container_flags='',
):
"""
Runs a shell command using doodad via a specified launch mode.
Args:
command (str): A shell command
cli_args (str): Command line args to pass
mode (LaunchMode): A LaunchMode object
mounts (tuple): A list/tuple of Mount objects
return_output (bool): If True, returns stdout as a string.
Do not use if the output will be large.
container_type (string): either 'docker' or 'singularity
extra_container_flags (string): flags other than GPU flags to pass to
docker run or singularity exec.
Returns:
A string output if return_output is True,
else None
"""
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(archive_filename=archive_file,
payload_script=command,
verbose=False,
docker_image=docker_image,
singularity_image=singularity_image,
container_type=container_type,
extra_container_flags=extra_container_flags,
use_gpu_image=mode.use_gpu,
mounts=mounts)
cmd = archive
if cli_args:
cmd = archive + ' -- ' + cli_args
result = mode.run_script(cmd, return_output=return_output, verbose=verbose)
if return_output:
result = archive_builder._strip_stdout(result)
return result
def test_s3(self):
""" Dry-run test for MountS3
This test doesn't actually run things it just makes sure nothing errors.
(we don't actually want to launch a test on EC2 and spend money)
"""
mnts = []
mnts.append(mount.MountS3(
s3_path='logs/mylogs',
dry=True,
))
payload_script = 'echo hello123'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=False,
docker_image='python:3',
mounts=mnts)
def test_local_repo(self):
mnts = []
mnts.append(
mount.MountLocal(local_dir=TESTING_DIR,
mount_point='./mymount',
output=False))
payload_script = 'cat ./mymount/secret.txt'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=False,
docker_image='python:3',
mounts=mnts)
output, errors = archive_builder_docker.run_archive(archive, timeout=5)
output = output.strip()
self.assertEqual(output, 'apple')
def test_git_repo(self):
mnts = []
mnts.append(
mount.MountGit(git_url='https://github.com/justinjfu/doodad.git',
branch='archive_builder_test',
mount_point='./code/doodad'))
payload_script = 'python3 ./code/doodad/test/hello_world.py'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=False,
docker_image='python:3',
mounts=mnts)
output, errors = archive_builder_docker.run_archive(archive, timeout=5)
output = output.strip()
self.assertEqual(output, 'hello world!')
def test_git_repo_pythonpath(self):
mnts = []
mnts.append(
mount.MountGit(git_url='https://github.com/justinjfu/doodad.git',
branch='archive_builder_test',
ssh_identity='~/.ssh/github',
mount_point='./code/doodad',
pythonpath=True))
payload_script = 'python3 ./code/doodad/test/test_import.py'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=False,
docker_image='python:3',
mounts=mnts)
output, errors = archive_builder_docker.run_archive(archive, timeout=5)
output = output.strip()
self.assertEqual(output, 'apple')
def run_command(command,
cli_args=None,
mode=launch_mode.LocalMode(),
mounts=tuple(),
return_output=False,
verbose=False,
docker_image='ubuntu:18.04'):
"""
Runs a shell command using doodad via a specified launch mode.
Args:
command (str): A shell command
cli_args (str): Command line args to pass
mode (LaunchMode): A LaunchMode object
mounts (tuple): A list/tuple of Mount objects
return_output (bool): If True, returns stdout as a string.
Do not use if the output will be large.
Returns:
A string output if return_output is True,
else None
"""
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(archive_filename=archive_file,
payload_script=command,
verbose=False,
docker_image=docker_image,
use_nvidia_docker=mode.use_gpu,
mounts=mounts)
cmd = archive
if cli_args:
cmd = archive + ' -- ' + cli_args
result = mode.run_script(cmd,
return_output=return_output,
verbose=verbose)
if return_output:
result = archive_builder._strip_stdout(result)
return result
def test_local_output(self):
mnts = []
try:
shutil.rmtree(TESTING_OUTPUT_DIR)
except OSError:
pass
temp_dir = TESTING_OUTPUT_DIR
os.makedirs(TESTING_OUTPUT_DIR)
mnts.append(
mount.MountLocal(local_dir=temp_dir,
mount_point='/mymount',
output=True))
payload_script = 'echo hello123 > /mymount/secret.txt'
archive = archive_builder_docker.build_archive(
payload_script=payload_script,
verbose=False,
docker_image='python:3',
mounts=mnts)
archive_builder_docker.run_archive(archive,
timeout=5,
get_output=False)
with open(path.join(temp_dir, 'secret.txt'), 'r') as f:
output = f.read()
self.assertEqual(output, 'hello123\n')
