def launch_command(self, cmd, mount_points=None, dry=False, verbose=False):
full_cmd = self.create_slurm_command(
cmd, mount_points=mount_points,
)
utils.call_and_wait(
full_cmd, verbose=verbose, dry=dry, skip_wait=self.skip_wait
)
def launch_command(self,
cmd,
mount_points=None,
dry=False,
verbose=False,
port=None):
mnt_args = ''
py_path = []
for mount in mount_points:
if isinstance(mount, MountLocal):
# mount_pnt = os.path.expanduser(mount.mount_point)
mount_pnt = mount.mount_dir()
mnt_args += ' -v %s:%s' % (mount.local_dir, mount_pnt)
utils.call_and_wait('mkdir -p %s' % mount.local_dir)
if mount.pythonpath:
py_path.append(mount_pnt)
else:
raise NotImplementedError(type(mount))
full_cmd = self.get_docker_cmd(cmd,
extra_args=mnt_args,
pythonpath=py_path,
checkpoint=self.checkpoints,
port=port)
utils.call_and_wait(full_cmd,
verbose=verbose,
dry=dry,
skip_wait=self.skip_wait)
def launch_command(self, main_cmd, mount_points=None, dry=False,
verbose=False):
py_path = []
remote_cmds = utils.CommandBuilder()
remote_cleanup_commands = utils.CommandBuilder()
mnt_args = ''
tmp_dir_cmd = 'mkdir -p %s' % self.tmp_dir
tmp_dir_cmd = self.credentials.get_ssh_bash_cmd(tmp_dir_cmd)
utils.call_and_wait(tmp_dir_cmd, dry=dry, verbose=verbose)
# remove_tmp_dir_cmd = 'rm -r %s' % self.tmp_dir
# remote_cleanup_commands.append(remove_tmp_dir_cmd)
# SCP Code over
for mount in mount_points:
if isinstance(mount, MountLocal):
if mount.read_only:
with mount.gzip() as gzip_file:
# scp
base_name = os.path.basename(gzip_file)
# file_hash = hash_file(gzip_path) # TODO: store all code in a special "caches" folder
remote_mnt_dir = os.path.join(self.tmp_dir,
os.path.splitext(
base_name)[0])
remote_tar = os.path.join(self.tmp_dir, base_name)
scp_cmd = self.credentials.get_scp_cmd(gzip_file,
remote_tar)
utils.call_and_wait(scp_cmd, dry=dry, verbose=verbose)
remote_cmds.append('mkdir -p %s' % remote_mnt_dir)
unzip_cmd = 'tar -xf %s -C %s' % (remote_tar, remote_mnt_dir)
remote_cmds.append(unzip_cmd)
remove_tar_cmd = 'rm %s' % remote_tar
remote_cmds.append(remove_tar_cmd)
mount_point = mount.mount_dir()
mnt_args += ' -B %s:%s' % (os.path.join(remote_mnt_dir,
os.path.basename(
mount.local_dir)),
mount_point)
else:
# remote_cmds.append('mkdir -p %s' % mount.mount_point)
remote_cmds.append('mkdir -p %s' % mount.local_dir_raw)
mnt_args += ' -B %s:%s' % (
mount.local_dir_raw, mount.mount_point)
if mount.pythonpath:
py_path.append(mount_point)
else:
raise NotImplementedError()
singularity_cmd = self.get_singularity_cmd(main_cmd,
extra_args=mnt_args,
pythonpath=py_path)
remote_cmds.append(singularity_cmd)
remote_cmds.extend(remote_cleanup_commands)
with tempfile.NamedTemporaryFile('w+', suffix='.sh') as ntf:
for cmd in remote_cmds:
if verbose:
ntf.write(
'echo "%s$ %s"\n' % (self.credentials.user_host, cmd))
ntf.write(cmd + '\n')
ntf.seek(0)
ssh_cmd = self.credentials.get_ssh_script_cmd(ntf.name)
utils.call_and_wait(ssh_cmd, dry=dry, verbose=verbose)
