def submit(self): """ submit the tracking job using slurm """ with change_directory(self.folder): job_id = None #< job_id of the previous process # submit new jobs for pass_id in self.passes: # create job command cmd = ['sbatch'] if job_id is not None: cmd.append('--dependency=afterok:%d' % job_id) cmd.append(self.files_job[pass_id][0]) # submit command and fetch job_id from output try: res = sp.check_output(cmd) except sp.CalledProcessError as err: # output the error message if there is any self.logger.error(err.output) raise job_id = int(res.split()[-1]) # save job_id to file with open(self.job_ids_file, 'a') as f: f.write('pass %d - %d\n' % (pass_id, job_id)) self.logger.info('Job id of pass %d: %d', pass_id, job_id)
def submit_underground_videos(self, job_parts): """ submit a slurm job array for creating the underground video. The `job_parts` determine which jobs will be submitted as a job array. This uses the slurm job array syntax, i.e. ranges should be given as `0-10`, and lists can be supplied by `1,3,6` """ with change_directory(self.folder): ensure_directory_exists('underground_video') # prepare the submission cmd = ['sbatch', '--array=%s' % job_parts, 'make_underground_videos.sh'] # submit command and fetch job_id from output try: res = sp.check_output(cmd) except sp.CalledProcessError as err: # output the error message if there is any self.logger.error(err.output) raise job_id = int(res.split()[-1]) # save job_id to file with open(self.job_ids_file, 'a') as f: f.write('pass 8 - %d\n' % job_id) self.logger.info('Job id of pass 8: %d', job_id)
def submit(self): """ submit the tracking job using slurm """ with change_directory(self.folder): job_id = None #< job_id of the previous process # submit new jobs for pass_id in self.passes: # create job command cmd = ['sbatch'] if job_id is not None: cmd.append('--dependency=afterok:%d' % job_id) cmd.append(self.files_job[pass_id][0]) # submit command and fetch job_id from output res = sp.check_output(cmd) job_id = int(res.split()[-1]) # save job_id to file with open(self.job_ids_file, 'a') as f: f.write('pass %d - %d\n' % (pass_id, job_id)) self.logger.info('Job id of pass %d: %d', pass_id, job_id)
def get_code_status(self): """ returns a dictionary with information about the current version of the code in the local git repository """ code_status = {} def get_output(cmd): try: output = subprocess.check_output(cmd, stderr=DEVNULL) except (OSError, subprocess.CalledProcessError): output = None return output # go to root of project folder, _ = os.path.split(__file__) folder = os.path.abspath(os.path.join(folder, '..')) with change_directory(folder): # get number of commits commit_count = get_output(['git', 'rev-list', 'HEAD', '--count']) if commit_count is None: output = get_output(['git', 'rev-list', 'HEAD', '--count']) if output is not None: commit_count = int(output.count('\n')) else: commit_count = int(commit_count.strip()) code_status['commit_count'] = commit_count # get the current revision revision = get_output(['git', 'rev-parse', 'HEAD']) if revision is not None: revision = revision.splitlines()[0] code_status['revision'] = revision # get the date of the last change last_change = get_output(['git', 'show', '-s', r'--format=%ci']) if last_change is not None: last_change = last_change.splitlines()[0] code_status['last_change'] = last_change return code_status