def launch_command(self, cmd, mount_points=None, dry=False, verbose=False): full_cmd = self.create_slurm_command( cmd, mount_points=mount_points, ) utils.call_and_wait( full_cmd, verbose=verbose, dry=dry, skip_wait=self.skip_wait )
def launch_command(self, cmd, mount_points=None, dry=False, verbose=False, port=None): mnt_args = '' py_path = [] for mount in mount_points: if isinstance(mount, MountLocal): # mount_pnt = os.path.expanduser(mount.mount_point) mount_pnt = mount.mount_dir() mnt_args += ' -v %s:%s' % (mount.local_dir, mount_pnt) utils.call_and_wait('mkdir -p %s' % mount.local_dir) if mount.pythonpath: py_path.append(mount_pnt) else: raise NotImplementedError(type(mount)) full_cmd = self.get_docker_cmd(cmd, extra_args=mnt_args, pythonpath=py_path, checkpoint=self.checkpoints, port=port) utils.call_and_wait(full_cmd, verbose=verbose, dry=dry, skip_wait=self.skip_wait)
def launch_command(self, main_cmd, mount_points=None, dry=False, verbose=False): py_path = [] remote_cmds = utils.CommandBuilder() remote_cleanup_commands = utils.CommandBuilder() mnt_args = '' tmp_dir_cmd = 'mkdir -p %s' % self.tmp_dir tmp_dir_cmd = self.credentials.get_ssh_bash_cmd(tmp_dir_cmd) utils.call_and_wait(tmp_dir_cmd, dry=dry, verbose=verbose) # remove_tmp_dir_cmd = 'rm -r %s' % self.tmp_dir # remote_cleanup_commands.append(remove_tmp_dir_cmd) # SCP Code over for mount in mount_points: if isinstance(mount, MountLocal): if mount.read_only: with mount.gzip() as gzip_file: # scp base_name = os.path.basename(gzip_file) # file_hash = hash_file(gzip_path) # TODO: store all code in a special "caches" folder remote_mnt_dir = os.path.join(self.tmp_dir, os.path.splitext( base_name)[0]) remote_tar = os.path.join(self.tmp_dir, base_name) scp_cmd = self.credentials.get_scp_cmd(gzip_file, remote_tar) utils.call_and_wait(scp_cmd, dry=dry, verbose=verbose) remote_cmds.append('mkdir -p %s' % remote_mnt_dir) unzip_cmd = 'tar -xf %s -C %s' % (remote_tar, remote_mnt_dir) remote_cmds.append(unzip_cmd) remove_tar_cmd = 'rm %s' % remote_tar remote_cmds.append(remove_tar_cmd) mount_point = mount.mount_dir() mnt_args += ' -B %s:%s' % (os.path.join(remote_mnt_dir, os.path.basename( mount.local_dir)), mount_point) else: # remote_cmds.append('mkdir -p %s' % mount.mount_point) remote_cmds.append('mkdir -p %s' % mount.local_dir_raw) mnt_args += ' -B %s:%s' % ( mount.local_dir_raw, mount.mount_point) if mount.pythonpath: py_path.append(mount_point) else: raise NotImplementedError() singularity_cmd = self.get_singularity_cmd(main_cmd, extra_args=mnt_args, pythonpath=py_path) remote_cmds.append(singularity_cmd) remote_cmds.extend(remote_cleanup_commands) with tempfile.NamedTemporaryFile('w+', suffix='.sh') as ntf: for cmd in remote_cmds: if verbose: ntf.write( 'echo "%s$ %s"\n' % (self.credentials.user_host, cmd)) ntf.write(cmd + '\n') ntf.seek(0) ssh_cmd = self.credentials.get_ssh_script_cmd(ntf.name) utils.call_and_wait(ssh_cmd, dry=dry, verbose=verbose)