def get_singularity_cmd(self, main_cmd, extra_args='', verbose=True, pythonpath=None, pre_cmd=None, post_cmd=None): cmd_list = utils.CommandBuilder() if pre_cmd: cmd_list.extend(pre_cmd) if verbose: if self.gpu: cmd_list.append('echo \"Running in singularity with gpu\"') else: cmd_list.append('echo \"Running in singularity\"') if pythonpath: cmd_list.append( 'export PYTHONPATH=$PYTHONPATH:%s' % (':'.join(pythonpath))) cmd_list.append(main_cmd) if post_cmd: cmd_list.extend(post_cmd) extra_args += ' -c ' extra_args += ' --no-home ' extra_args += ' --cleanenv ' extra_args += ' --writable-tmpfs ' if self.gpu: extra_args += ' --nv ' singularity_prefix = self.singularity_bin + ' instance start %s %s %s /bin/bash -c ' % ( extra_args, self.singularity_image, self.singularity_name) main_cmd = cmd_list.to_string() full_cmd = singularity_prefix + ("\'%s\'" % main_cmd) return full_cmd
def create_slurm_command(self, main_cmd, mount_points=None): config = self._slurm_config pre_cmd = SingularityMode.build_pre_cmd(self, mount_points=mount_points).to_string('\n') command = utils.CommandBuilder() command.append(main_cmd) if self.post_cmd is not None: command.append(self.post_cmd) command = command.to_string('\n') n_tasks = 1 num_cpus = n_tasks * config.n_cpus_per_task n_nodes = config.n_nodes or math.ceil( num_cpus / config.max_num_cores_per_node ) script_file = create_ray_slurm_script( job_name=config.job_name, command=command, num_gpus=config.n_gpus, partition=config.partition, num_nodes=n_nodes, n_cpus_per_task=config.n_cpus_per_task, time_in_mins=config.time_in_mins, load_env=pre_cmd, extra_flags=config.extra_flags, base_log_dir=config.base_log_dir, ) full_cmd = f"sbatch {script_file}" return full_cmd
def launch_command( self, cmd, dry=False, mount_points=None, verbose=False, ): full_cmd = self.create_singularity_cmd(cmd, mount_points=mount_points) utils.add_to_script( full_cmd, path=self.TASK_FILE, verbose=True, overwrite=self._overwrite_task_script, ) cmd_list = utils.CommandBuilder() cmd_list.append('module load gcc openmpi') cmd_list.append('ht_helper.sh -m "python/3.5" -t {}'.format( self._taskfile_path_on_brc)) sbatch_cmd = slurm_util.wrap_command_with_sbatch( cmd_list.to_string(), self._slurm_config, self._n_tasks_total, ) utils.add_to_script( sbatch_cmd, path=self.SBATCH_FILE, verbose=True, overwrite=True, )
def get_docker_cmd(self, main_cmd, extra_args='', use_tty=True, verbose=True, pythonpath=None, pre_cmd=None, post_cmd=None, checkpoint=False, no_root=False, use_docker_generated_name=False): cmd_list = utils.CommandBuilder() if pre_cmd: cmd_list.extend(pre_cmd) if verbose: if self.gpu: cmd_list.append('echo \"Running in docker with gpu\"') else: cmd_list.append('echo \"Running in docker\"') if pythonpath: cmd_list.append('export PYTHONPATH=$PYTHONPATH:%s' % (':'.join(pythonpath))) if no_root: # This should work if you're running a script # cmd_list.append('useradd --uid $(id -u) --no-create-home --home-dir / doodaduser') # cmd_list.append('su - doodaduser /bin/bash {script}') # this is a temp workaround extra_args += ' -u $(id -u)' cmd_list.append(main_cmd) if post_cmd: cmd_list.extend(post_cmd) docker_name = self.docker_name if docker_name and not use_docker_generated_name: extra_args += ' --name %s ' % docker_name if checkpoint: # set up checkpoint stuff use_tty = False extra_args += ' -d ' # detach is optional if self.gpu: docker_run = 'docker run --gpus all' else: docker_run = 'docker run' if use_tty: docker_prefix = '%s %s -ti %s /bin/bash -c ' % ( docker_run, extra_args, self.docker_image) else: docker_prefix = '%s %s %s /bin/bash -c ' % (docker_run, extra_args, self.docker_image) main_cmd = cmd_list.to_string() full_cmd = docker_prefix + ("\'%s\'" % main_cmd) return full_cmd
def launch_command(self, cmd, mount_points=None, dry=False, verbose=False): if dry: print(cmd) return commands = utils.CommandBuilder() # chdir to home dir commands.append('cd %s' % (os.path.expanduser('~'))) # do mounting py_path = [] cleanup_commands = utils.CommandBuilder() for mount in mount_points: print('mounting:', mount) if isinstance(mount, MountLocal): if not mount.no_remount: mount.create_if_nonexistent() commands.append('ln -s %s %s' % (mount.local_dir, mount.mount_point)) # subprocess.call(symlink_cmd, shell=True) if mount.cleanup: cleanup_commands.append('rm "%s"' % mount.mount_point) if mount.pythonpath: py_path.append(mount.mount_point) else: raise NotImplementedError() # add pythonpath mounts if py_path: commands.append('export PYTHONPATH=$PYTHONPATH:%s' % (':'.join(py_path))) # Add main command commands.append(cmd) # cleanup commands.extend(cleanup_commands) # Call everything commands.call_and_wait(verbose=verbose, dry=dry, skip_wait=self.skip_wait)
def create_singularity_cmd( self, main_cmd, mount_points=None, ): extra_args = self._extra_args cmd_list = utils.CommandBuilder() if self.pre_cmd: cmd_list.extend(self.pre_cmd) if self._verbose_cmd: if self.gpu: cmd_list.append('echo \"Running in singularity (gpu)\"') else: cmd_list.append('echo \"Running in singularity\"') py_paths = [] for mount in mount_points: if isinstance(mount, MountLocal): if mount.pythonpath: py_paths.append(mount.local_dir) else: raise NotImplementedError(type(mount)) if py_paths: cmd_list.append('export PYTHONPATH=$PYTHONPATH:%s' % (':'.join(py_paths))) cmd_list.append(main_cmd) if self.post_cmd: cmd_list.extend(self.post_cmd) if self.gpu: extra_args += ' --nv ' if isinstance(extra_args, list): extra_args = ''.join(extra_args) singularity_prefix = 'singularity exec %s %s /bin/bash -c ' % ( extra_args, self.singularity_image, ) main_cmd = cmd_list.to_string() full_cmd = singularity_prefix + ("\'%s\'" % main_cmd) return full_cmd
def build_pre_cmd(self, mount_points) -> utils.CommandBuilder: cmd_list = utils.CommandBuilder() if self.pre_cmd: cmd_list.extend(self.pre_cmd) if self._verbose_cmd: if self.gpu: cmd_list.append('echo \"Running in singularity (gpu)\"') else: cmd_list.append('echo \"Running in singularity\"') py_paths = [] for mount in mount_points: if isinstance(mount, MountLocal): if mount.pythonpath: py_paths.append(mount.local_dir) else: raise NotImplementedError(type(mount)) if py_paths: cmd_list.append('export PYTHONPATH=$PYTHONPATH:%s' % ( ':'.join(py_paths))) return cmd_list
def launch_command(self, main_cmd, mount_points=None, dry=False, verbose=False): py_path = [] remote_cmds = utils.CommandBuilder() remote_cleanup_commands = utils.CommandBuilder() mnt_args = '' tmp_dir_cmd = 'mkdir -p %s' % self.tmp_dir tmp_dir_cmd = self.credentials.get_ssh_bash_cmd(tmp_dir_cmd) utils.call_and_wait(tmp_dir_cmd, dry=dry, verbose=verbose) # remove_tmp_dir_cmd = 'rm -r %s' % self.tmp_dir # remote_cleanup_commands.append(remove_tmp_dir_cmd) # SCP Code over for mount in mount_points: if isinstance(mount, MountLocal): if mount.read_only: with mount.gzip() as gzip_file: # scp base_name = os.path.basename(gzip_file) # file_hash = hash_file(gzip_path) # TODO: store all code in a special "caches" folder remote_mnt_dir = os.path.join(self.tmp_dir, os.path.splitext( base_name)[0]) remote_tar = os.path.join(self.tmp_dir, base_name) scp_cmd = self.credentials.get_scp_cmd(gzip_file, remote_tar) utils.call_and_wait(scp_cmd, dry=dry, verbose=verbose) remote_cmds.append('mkdir -p %s' % remote_mnt_dir) unzip_cmd = 'tar -xf %s -C %s' % (remote_tar, remote_mnt_dir) remote_cmds.append(unzip_cmd) remove_tar_cmd = 'rm %s' % remote_tar remote_cmds.append(remove_tar_cmd) mount_point = mount.mount_dir() mnt_args += ' -B %s:%s' % (os.path.join(remote_mnt_dir, os.path.basename( mount.local_dir)), mount_point) else: # remote_cmds.append('mkdir -p %s' % mount.mount_point) remote_cmds.append('mkdir -p %s' % mount.local_dir_raw) mnt_args += ' -B %s:%s' % ( mount.local_dir_raw, mount.mount_point) if mount.pythonpath: py_path.append(mount_point) else: raise NotImplementedError() singularity_cmd = self.get_singularity_cmd(main_cmd, extra_args=mnt_args, pythonpath=py_path) remote_cmds.append(singularity_cmd) remote_cmds.extend(remote_cleanup_commands) with tempfile.NamedTemporaryFile('w+', suffix='.sh') as ntf: for cmd in remote_cmds: if verbose: ntf.write( 'echo "%s$ %s"\n' % (self.credentials.user_host, cmd)) ntf.write(cmd + '\n') ntf.seek(0) ssh_cmd = self.credentials.get_ssh_script_cmd(ntf.name) utils.call_and_wait(ssh_cmd, dry=dry, verbose=verbose)
def get_docker_cmd( self, main_cmd, extra_args="", use_tty=True, verbose=True, pythonpath=None, pre_cmd=None, post_cmd=None, checkpoint=False, no_root=False, use_docker_generated_name=False, ): cmd_list = utils.CommandBuilder() if pre_cmd: cmd_list.extend(pre_cmd) if verbose: if self.gpu: cmd_list.append('echo "Running in docker with gpu"') else: cmd_list.append('echo "Running in docker"') if pythonpath: cmd_list.append("export PYTHONPATH=$PYTHONPATH:%s" % (":".join(pythonpath))) if no_root: # This should work if you're running a script # cmd_list.append('useradd --uid $(id -u) --no-create-home --home-dir / doodaduser') # cmd_list.append('su - doodaduser /bin/bash {script}') # this is a temp workaround extra_args += " -u $(id -u)" cmd_list.append(main_cmd) if post_cmd: cmd_list.extend(post_cmd) docker_name = self.docker_name if docker_name and not use_docker_generated_name: extra_args += " --name %s " % docker_name if checkpoint: # set up checkpoint stuff use_tty = False extra_args += " -d " # detach is optional if self.gpu: docker_run = "docker run --gpus device={}".format(self.gpu_id) else: docker_run = "docker run" if use_tty: docker_prefix = "%s %s -ti %s /bin/bash -c " % ( docker_run, extra_args, self.docker_image, ) else: docker_prefix = "%s %s %s /bin/bash -c " % ( docker_run, extra_args, self.docker_image, ) main_cmd = cmd_list.to_string() full_cmd = docker_prefix + ("'%s'" % main_cmd) return full_cmd