def get_singularity_cmd(self, main_cmd, extra_args='',
verbose=True, pythonpath=None, pre_cmd=None,
post_cmd=None):
cmd_list = utils.CommandBuilder()
if pre_cmd:
cmd_list.extend(pre_cmd)
if verbose:
if self.gpu:
cmd_list.append('echo \"Running in singularity with gpu\"')
else:
cmd_list.append('echo \"Running in singularity\"')
if pythonpath:
cmd_list.append(
'export PYTHONPATH=$PYTHONPATH:%s' % (':'.join(pythonpath)))
cmd_list.append(main_cmd)
if post_cmd:
cmd_list.extend(post_cmd)
extra_args += ' -c '
extra_args += ' --no-home '
extra_args += ' --cleanenv '
extra_args += ' --writable-tmpfs '
if self.gpu:
extra_args += ' --nv '
singularity_prefix = self.singularity_bin + ' instance start %s %s %s /bin/bash -c ' % (
extra_args, self.singularity_image, self.singularity_name)
main_cmd = cmd_list.to_string()
full_cmd = singularity_prefix + ("\'%s\'" % main_cmd)
return full_cmd
def create_slurm_command(self, main_cmd, mount_points=None):
config = self._slurm_config
pre_cmd = SingularityMode.build_pre_cmd(self, mount_points=mount_points).to_string('\n')
command = utils.CommandBuilder()
command.append(main_cmd)
if self.post_cmd is not None:
command.append(self.post_cmd)
command = command.to_string('\n')
n_tasks = 1
num_cpus = n_tasks * config.n_cpus_per_task
n_nodes = config.n_nodes or math.ceil(
num_cpus / config.max_num_cores_per_node
)
script_file = create_ray_slurm_script(
job_name=config.job_name,
command=command,
num_gpus=config.n_gpus,
partition=config.partition,
num_nodes=n_nodes,
n_cpus_per_task=config.n_cpus_per_task,
time_in_mins=config.time_in_mins,
load_env=pre_cmd,
extra_flags=config.extra_flags,
base_log_dir=config.base_log_dir,
)
full_cmd = f"sbatch {script_file}"
return full_cmd
def launch_command(
self,
cmd,
dry=False,
mount_points=None,
verbose=False,
):
full_cmd = self.create_singularity_cmd(cmd, mount_points=mount_points)
utils.add_to_script(
full_cmd,
path=self.TASK_FILE,
verbose=True,
overwrite=self._overwrite_task_script,
)
cmd_list = utils.CommandBuilder()
cmd_list.append('module load gcc openmpi')
cmd_list.append('ht_helper.sh -m "python/3.5" -t {}'.format(
self._taskfile_path_on_brc))
sbatch_cmd = slurm_util.wrap_command_with_sbatch(
cmd_list.to_string(),
self._slurm_config,
self._n_tasks_total,
)
utils.add_to_script(
sbatch_cmd,
path=self.SBATCH_FILE,
verbose=True,
overwrite=True,
)
def get_docker_cmd(self,
main_cmd,
extra_args='',
use_tty=True,
verbose=True,
pythonpath=None,
pre_cmd=None,
post_cmd=None,
checkpoint=False,
no_root=False,
use_docker_generated_name=False):
cmd_list = utils.CommandBuilder()
if pre_cmd:
cmd_list.extend(pre_cmd)
if verbose:
if self.gpu:
cmd_list.append('echo \"Running in docker with gpu\"')
else:
cmd_list.append('echo \"Running in docker\"')
if pythonpath:
cmd_list.append('export PYTHONPATH=$PYTHONPATH:%s' %
(':'.join(pythonpath)))
if no_root:
# This should work if you're running a script
# cmd_list.append('useradd --uid $(id -u) --no-create-home --home-dir / doodaduser')
# cmd_list.append('su - doodaduser /bin/bash {script}')
# this is a temp workaround
extra_args += ' -u $(id -u)'
cmd_list.append(main_cmd)
if post_cmd:
cmd_list.extend(post_cmd)
docker_name = self.docker_name
if docker_name and not use_docker_generated_name:
extra_args += ' --name %s ' % docker_name
if checkpoint:
# set up checkpoint stuff
use_tty = False
extra_args += ' -d ' # detach is optional
if self.gpu:
docker_run = 'docker run --gpus all'
else:
docker_run = 'docker run'
if use_tty:
docker_prefix = '%s %s -ti %s /bin/bash -c ' % (
docker_run, extra_args, self.docker_image)
else:
docker_prefix = '%s %s %s /bin/bash -c ' % (docker_run, extra_args,
self.docker_image)
main_cmd = cmd_list.to_string()
full_cmd = docker_prefix + ("\'%s\'" % main_cmd)
return full_cmd
def launch_command(self, cmd, mount_points=None, dry=False, verbose=False):
if dry:
print(cmd)
return
commands = utils.CommandBuilder()
# chdir to home dir
commands.append('cd %s' % (os.path.expanduser('~')))
# do mounting
py_path = []
cleanup_commands = utils.CommandBuilder()
for mount in mount_points:
print('mounting:', mount)
if isinstance(mount, MountLocal):
if not mount.no_remount:
mount.create_if_nonexistent()
commands.append('ln -s %s %s' %
(mount.local_dir, mount.mount_point))
# subprocess.call(symlink_cmd, shell=True)
if mount.cleanup:
cleanup_commands.append('rm "%s"' % mount.mount_point)
if mount.pythonpath:
py_path.append(mount.mount_point)
else:
raise NotImplementedError()
# add pythonpath mounts
if py_path:
commands.append('export PYTHONPATH=$PYTHONPATH:%s' %
(':'.join(py_path)))
# Add main command
commands.append(cmd)
# cleanup
commands.extend(cleanup_commands)
# Call everything
commands.call_and_wait(verbose=verbose,
dry=dry,
skip_wait=self.skip_wait)
def create_singularity_cmd(
self,
main_cmd,
mount_points=None,
):
extra_args = self._extra_args
cmd_list = utils.CommandBuilder()
if self.pre_cmd:
cmd_list.extend(self.pre_cmd)
if self._verbose_cmd:
if self.gpu:
cmd_list.append('echo \"Running in singularity (gpu)\"')
else:
cmd_list.append('echo \"Running in singularity\"')
py_paths = []
for mount in mount_points:
if isinstance(mount, MountLocal):
if mount.pythonpath:
py_paths.append(mount.local_dir)
else:
raise NotImplementedError(type(mount))
if py_paths:
cmd_list.append('export PYTHONPATH=$PYTHONPATH:%s' %
(':'.join(py_paths)))
cmd_list.append(main_cmd)
if self.post_cmd:
cmd_list.extend(self.post_cmd)
if self.gpu:
extra_args += ' --nv '
if isinstance(extra_args, list):
extra_args = ''.join(extra_args)
singularity_prefix = 'singularity exec %s %s /bin/bash -c ' % (
extra_args,
self.singularity_image,
)
main_cmd = cmd_list.to_string()
full_cmd = singularity_prefix + ("\'%s\'" % main_cmd)
return full_cmd
def build_pre_cmd(self, mount_points) -> utils.CommandBuilder:
cmd_list = utils.CommandBuilder()
if self.pre_cmd:
cmd_list.extend(self.pre_cmd)
if self._verbose_cmd:
if self.gpu:
cmd_list.append('echo \"Running in singularity (gpu)\"')
else:
cmd_list.append('echo \"Running in singularity\"')
py_paths = []
for mount in mount_points:
if isinstance(mount, MountLocal):
if mount.pythonpath:
py_paths.append(mount.local_dir)
else:
raise NotImplementedError(type(mount))
if py_paths:
cmd_list.append('export PYTHONPATH=$PYTHONPATH:%s' % (
':'.join(py_paths)))
return cmd_list
def launch_command(self, main_cmd, mount_points=None, dry=False,
verbose=False):
py_path = []
remote_cmds = utils.CommandBuilder()
remote_cleanup_commands = utils.CommandBuilder()
mnt_args = ''
tmp_dir_cmd = 'mkdir -p %s' % self.tmp_dir
tmp_dir_cmd = self.credentials.get_ssh_bash_cmd(tmp_dir_cmd)
utils.call_and_wait(tmp_dir_cmd, dry=dry, verbose=verbose)
# remove_tmp_dir_cmd = 'rm -r %s' % self.tmp_dir
# remote_cleanup_commands.append(remove_tmp_dir_cmd)
# SCP Code over
for mount in mount_points:
if isinstance(mount, MountLocal):
if mount.read_only:
with mount.gzip() as gzip_file:
# scp
base_name = os.path.basename(gzip_file)
# file_hash = hash_file(gzip_path) # TODO: store all code in a special "caches" folder
remote_mnt_dir = os.path.join(self.tmp_dir,
os.path.splitext(
base_name)[0])
remote_tar = os.path.join(self.tmp_dir, base_name)
scp_cmd = self.credentials.get_scp_cmd(gzip_file,
remote_tar)
utils.call_and_wait(scp_cmd, dry=dry, verbose=verbose)
remote_cmds.append('mkdir -p %s' % remote_mnt_dir)
unzip_cmd = 'tar -xf %s -C %s' % (remote_tar, remote_mnt_dir)
remote_cmds.append(unzip_cmd)
remove_tar_cmd = 'rm %s' % remote_tar
remote_cmds.append(remove_tar_cmd)
mount_point = mount.mount_dir()
mnt_args += ' -B %s:%s' % (os.path.join(remote_mnt_dir,
os.path.basename(
mount.local_dir)),
mount_point)
else:
# remote_cmds.append('mkdir -p %s' % mount.mount_point)
remote_cmds.append('mkdir -p %s' % mount.local_dir_raw)
mnt_args += ' -B %s:%s' % (
mount.local_dir_raw, mount.mount_point)
if mount.pythonpath:
py_path.append(mount_point)
else:
raise NotImplementedError()
singularity_cmd = self.get_singularity_cmd(main_cmd,
extra_args=mnt_args,
pythonpath=py_path)
remote_cmds.append(singularity_cmd)
remote_cmds.extend(remote_cleanup_commands)
with tempfile.NamedTemporaryFile('w+', suffix='.sh') as ntf:
for cmd in remote_cmds:
if verbose:
ntf.write(
'echo "%s$ %s"\n' % (self.credentials.user_host, cmd))
ntf.write(cmd + '\n')
ntf.seek(0)
ssh_cmd = self.credentials.get_ssh_script_cmd(ntf.name)
utils.call_and_wait(ssh_cmd, dry=dry, verbose=verbose)
def get_docker_cmd(
self,
main_cmd,
extra_args="",
use_tty=True,
verbose=True,
pythonpath=None,
pre_cmd=None,
post_cmd=None,
checkpoint=False,
no_root=False,
use_docker_generated_name=False,
):
cmd_list = utils.CommandBuilder()
if pre_cmd:
cmd_list.extend(pre_cmd)
if verbose:
if self.gpu:
cmd_list.append('echo "Running in docker with gpu"')
else:
cmd_list.append('echo "Running in docker"')
if pythonpath:
cmd_list.append("export PYTHONPATH=$PYTHONPATH:%s" %
(":".join(pythonpath)))
if no_root:
# This should work if you're running a script
# cmd_list.append('useradd --uid $(id -u) --no-create-home --home-dir / doodaduser')
# cmd_list.append('su - doodaduser /bin/bash {script}')
# this is a temp workaround
extra_args += " -u $(id -u)"
cmd_list.append(main_cmd)
if post_cmd:
cmd_list.extend(post_cmd)
docker_name = self.docker_name
if docker_name and not use_docker_generated_name:
extra_args += " --name %s " % docker_name
if checkpoint:
# set up checkpoint stuff
use_tty = False
extra_args += " -d " # detach is optional
if self.gpu:
docker_run = "docker run --gpus device={}".format(self.gpu_id)
else:
docker_run = "docker run"
if use_tty:
docker_prefix = "%s %s -ti %s /bin/bash -c " % (
docker_run,
extra_args,
self.docker_image,
)
else:
docker_prefix = "%s %s %s /bin/bash -c " % (
docker_run,
extra_args,
self.docker_image,
)
main_cmd = cmd_list.to_string()
full_cmd = docker_prefix + ("'%s'" % main_cmd)
return full_cmd
