def status(self): """ Get job status from HPC. Returns ------- status : (str) Job status. Possible status are 'PENDING','RUNNING','NOT IN QUEUE'. """ hpc = HPCInterface() stdout, stderr = hpc.qstat(printout=False) queue = stdout.splitlines() job_lines = grep_list(self.name, queue) if job_lines == []: status = 'NOT IN QUEUE' elif len(job_lines) > 1: raise ValueError( f'More than one job named "{self.name}" has been found in queue:\n{stdout}' ) else: job_line = job_lines[0].split() status = job_line[4] if status == 'PD': status = 'PENDING' if status == 'R': status = 'RUNNING' if status == 'CG': status = 'COMPLETED' return status
def cancel_job(self): """Cancel job on HPC""" hpc = HPCInterface() job_id = self.job_id() hpc.cancel_jobs(job_id) return
def run_job(self, write_input=True, sync=True): """ Run job on HPC. Input files are automatically written and sync to HPC is performed. Parameters ---------- write_input : (bool), optional Write input file stored in "inputs" dictionary. The default is True. sync : (bool), optional Sync files to HPC before running. The default is True Returns ------- stdout : (str) Output. stderr : (str) Error. """ if write_input: self.write_input() hpc = HPCInterface() if sync: self.sync_to_hpc() stdout, stderr = hpc.sbatch( path=self.path_in_hpc, job_script_filename=self.job_script_filename) return stdout, stderr
def sync_to_hpc(self, stdouts=False, exclude=None, dry_run=False): """ Sync job data from local machine to HPC Parameters ---------- stdouts : (bool), optional Return output and error strings. The default is False. exclude : (list), optional List of files to exclude in rsync. The default is None. dry_run : (bool), optional Perform dry run in rsync with --dry-run. The default is False. Returns ------- stdout : (str) Output. stderr : (str) Error. """ hpc = HPCInterface() abs_path = op.abspath(self.path) localdir = abs_path stdout, stderr = hpc.rsync_to_hpc(localdir=localdir, remotedir=self.path_in_hpc, exclude=exclude, dry_run=dry_run) if stdouts: return stdout, stderr else: return
def __init__(self,jobs=None,path=None,name=None,sort_by_name=True): """ Class to store sets of calculations Parameters ---------- jobs : (list), optional List of Job objects belonging to the dataset. The default is None. path : (str), optional Path of dataset directory. If None the work dir is used if jobs is None, else the commonpath between the jobs is used. The default is None. name : (str), optional Name to assign to dataset. The default is None. If None the folder name is used. sort_by_name : (bool), optional Sort list of jobs based on Job names. The default is True. """ if jobs: path = op.commonpath([j.path for j in jobs]) self.path = op.abspath(path) else: self.path = op.abspath(path) if path else os.getcwd() self.name = name if name else op.basename(self.path) self.sort_by_name = sort_by_name self.jobs = jobs if jobs: self._group_jobs() if sort_by_name: self.jobs = sorted(self.jobs, key=operator.attrgetter('name')) self._localdir = HPCInterface().localdir self._workdir = HPCInterface().workdir self._path_relative = self.path.replace(self._localdir,'') self.path_in_hpc = self._workdir + self._path_relative
def __init__(self, path=None, inputs=None, job_settings=None, outputs=None, job_script_filename=None, name=None): """ Class to control and organize inputs and outputs of a generic job. Parameters ---------- path : (str), optional Path where job is stored. The default is None. If None the work dir is used. inputs : (dict), optional Dictionary with input data. The default is None. job_settings : (dict), optional Dictionary with job settings. The default is None. Documentation in ScriptHandler class in slurm.job_script module outputs : (dict), optional Dictionary with output data. The default is None. job_script_filename : (str), optional Filename of job script. The default is taken from the key 'filename' in the job_settings in the config file. name : (str) Name of the job. If none the name is searched in the job script. """ self.path = path if path else os.getcwd() self.inputs = inputs self.job_settings = job_settings self.outputs = outputs self.job_script_filename = job_script_filename if job_script_filename else ScriptHandler( ).filename self._localdir = HPCInterface().localdir self._workdir = HPCInterface().workdir self._path_relative = op.abspath(self.path).replace(self._localdir, '') self.path_in_hpc = self._workdir + self._path_relative if outputs: self.get_output_properties() if name: self.name = name elif self.job_settings: self.name = self.job_settings['name'] elif op.isfile(op.join(self.path, self.job_script_filename)): s = ScriptHandler.from_file(self.path, filename=self.job_script_filename) self.name = s.settings['name'] else: self.name = 'no_name' if not self.job_settings: self.job_settings = {} self.job_settings['name'] = self.name
def job_queue(self): """ Print job queue from HPC on screen Returns ------- stdout : (str) Output. stderr : (str) Error. """ hpc = HPCInterface() stdout, stderr = hpc.qstat() return stdout, stderr
def queue(self,stdouts=False): """ Display queue from HPC. If stdouts is True returns out and err strings. Returns ------- stdout : (str) Output. stderr : (str) Error. """ hpc = HPCInterface() stdout,stderr = hpc.qstat() if stdouts: return stdout,stderr else: return
def job_id(self): """Get job ID from the queue on HPC""" hpc = HPCInterface() stdout, stderr = hpc.qstat(printout=False) queue = stdout.splitlines() job_lines = grep_list(self.name, queue) if job_lines == []: raise ValueError( f'Job named "{self.name}" is not currently running or pending') elif len(job_lines) > 1: raise ValueError( f'More than one job named "{self.name}" has been found in queue:\n{stdout}' ) else: job_line = job_lines[0].split() job_id = job_line[0] return job_id