def test_run_no_output(self):
"""Simple test to check for no errors
"""
launcher = mode.LocalMode()
result = launch_api.run_command('echo hello123',
mode=launcher,
return_output=False)
def test_python_args(self):
launcher = mode.LocalMode()
result = launch_api.run_python(target=path.join(
TESTING_DIR, 'argv.py'),
cli_args='--arg1 a --arg2 b',
mode=launcher,
return_output=True)
self.assertEqual(result.splitlines(), ['--arg1', 'a', '--arg2', 'b'])
def test_local(self):
launcher = mode.LocalMode()
with hello_script() as script_name:
output = launcher.run_script(script_name, return_output=True)
self.assertEqual(output, 'hello123\n')
output = launcher.run_script(script_name,
return_output=True,
dry=True)
def test_run_python(self):
launcher = mode.LocalMode()
result = launch_api.run_python(target=path.join(
TESTING_DIR, 'hello_world.py'),
mode=launcher,
return_output=True,
docker_image='python:3')
print(result)
self.assertEqual(result.strip(), 'hello123')
def run(args, run_mode='local', storage_mode='local', dry_run=False):
local_mount = mount.MountLocal(
local_dir="/home/ashwin/Research/playground/softlearning",
mount_point='/code',
output=False)
local_mount2 = mount.MountLocal(local_dir="/home/ashwin/.mujoco",
mount_point='/root/.mujoco',
output=False)
# you can either run code on your machine or in the cloud
if run_mode == 'local':
launcher = mode.LocalMode()
elif run_mode == 'ec2':
launcher = mode.EC2Autoconfig(
s3_bucket="doodad-ashwin-redone2",
#s3_log_path='test_doodad/softlearning_test',
s3_log_path='softlearning_exps/',
instance_type='c4.large',
spot_price=0.03,
region='us-west-1',
ami_name='ami-874378e7',
)
results_mount_point = '/root/ray_results'
if storage_mode == 'local':
output_mount = mount.MountLocal(
local_dir="/home/ashwin/Research/results/",
mount_point=results_mount_point,
output=True)
elif storage_mode == 's3':
output_mount = mount.MountS3(s3_path='softlearning_output',
mount_point=results_mount_point,
output=True)
mounts = [local_mount, local_mount2, output_mount]
kwargs = {
#'command': "cat /code/secret.txt > /root/ray_results/secret.txt",
'command': command(args.domain, args.environment, args.exp_name),
'cli_args': "",
'mode': launcher,
'mounts': mounts,
'verbose': True,
'docker_image': 'abhiguptaberk/softlearningashwin:latest'
}
print(kwargs)
if not dry_run:
launch_api.run_command(**kwargs)
def test_echo_args(self):
launcher = mode.LocalMode()
result = launch_api.run_command('echo',
cli_args='--help',
mode=launcher,
return_output=True)
self.assertEqual(result.strip(), '--help')
result = launch_api.run_command('echo',
cli_args='--help --help2',
mode=launcher,
return_output=True)
self.assertEqual(result.strip(), '--help --help2')
def run_command(
command,
cli_args=None,
mode=launch_mode.LocalMode(),
mounts=tuple(),
return_output=False,
verbose=False,
docker_image='ubuntu:18.04',
singularity_image=None,
container_type='docker',
extra_container_flags='',
):
"""
Runs a shell command using doodad via a specified launch mode.
Args:
command (str): A shell command
cli_args (str): Command line args to pass
mode (LaunchMode): A LaunchMode object
mounts (tuple): A list/tuple of Mount objects
return_output (bool): If True, returns stdout as a string.
Do not use if the output will be large.
container_type (string): either 'docker' or 'singularity
extra_container_flags (string): flags other than GPU flags to pass to
docker run or singularity exec.
Returns:
A string output if return_output is True,
else None
"""
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(archive_filename=archive_file,
payload_script=command,
verbose=False,
docker_image=docker_image,
singularity_image=singularity_image,
container_type=container_type,
extra_container_flags=extra_container_flags,
use_gpu_image=mode.use_gpu,
mounts=mounts)
cmd = archive
if cli_args:
cmd = archive + ' -- ' + cli_args
result = mode.run_script(cmd, return_output=return_output, verbose=verbose)
if return_output:
result = archive_builder._strip_stdout(result)
return result
def run_command(command,
cli_args=None,
mode=launch_mode.LocalMode(),
mounts=tuple(),
return_output=False,
verbose=False,
docker_image='ubuntu:18.04'):
"""
Runs a shell command using doodad via a specified launch mode.
Args:
command (str): A shell command
cli_args (str): Command line args to pass
mode (LaunchMode): A LaunchMode object
mounts (tuple): A list/tuple of Mount objects
return_output (bool): If True, returns stdout as a string.
Do not use if the output will be large.
Returns:
A string output if return_output is True,
else None
"""
with archive_builder.temp_archive_file() as archive_file:
archive = archive_builder.build_archive(archive_filename=archive_file,
payload_script=command,
verbose=False,
docker_image=docker_image,
use_nvidia_docker=mode.use_gpu,
mounts=mounts)
cmd = archive
if cli_args:
cmd = archive + ' -- ' + cli_args
result = mode.run_script(cmd,
return_output=return_output,
verbose=verbose)
if return_output:
result = archive_builder._strip_stdout(result)
return result
def test_mode(self):
launcher = mode.LocalMode(shell_interpreter='bashy')
self.assertEqual(
launcher._get_run_command('myscript.sh'),
'bashy myscript.sh'
)
def test_run_command(self):
launcher = mode.LocalMode()
result = launch_api.run_command('echo hello123',
mode=launcher,
return_output=True)
self.assertEqual(result.strip(), 'hello123')