def run_example(self): ensure_wdir(project=self.project) empty_queues(project=self.project) os.chdir(self.ex_dir) if os.path.exists('logfiles'): rmtree('logfiles') os.mkdir('logfiles') call('./create_jobs.py {}'.format(self.project), shell=True) call('fjd-recruiter --project {} hire {}'\ .format(self.project, self.num_workers), shell=True) call('fjd-dispatcher --project {} --end_when_jobs_are_done'\ .format(self.project), shell=True)
def __init__(self, exe, repeat=1, parameters=[], project=None, num_workers=0, callback=None, curdir=''): if not exe or exe == '': print('[fjd] Please specify an executable command (--exe).') sys.exit(2) if repeat > 1 and len(parameters) > 0: print('[fjd] Only one of --repeat and --parameters can be set at a time.') sys.exit(2) empty_queues(project=project) self.wdir = ensure_wdir(project) if len(parameters) > 1: for i, p in enumerate(parameters): job = '{}/jobqueue/job{}'.format(self.wdir, i) with open(job, 'w') as f: f.write('#!/bin/bash\n') cur_exe = exe ext_params = [] for j, param in enumerate(str(p).split('#')): if '${}'.format(j+1) in cur_exe: cur_exe = cur_exe.replace('${}'.format(j+1), str(param)) else: ext_params.append(str(p)) f.write('{exe} {params}'.format(exe=cur_exe, params=' '.join(ext_params))) os.chmod(job, 0o777) else: for i in range(repeat): job = '{}/jobqueue/job{}'.format(self.wdir, i) with open(job, 'w') as f: f.write('#!/bin/bash\n') f.write(exe) os.chmod(job, 0o777) if num_workers == 0: num_workers = cpu_count() - 1 num_workers = min(num_workers, cpu_count()) recruiter = Recruiter(num_workers=num_workers, project=project, curdir=curdir) recruiter.hire() Dispatcher(project=project, callback=callback)
''' import sys import os import itertools import numpy as np import random from subprocess import call from fjd import Dispatcher from fjd.utils import ensure_wdir, empty_queues # clean up call('rm pbsjobs/pbsjob*', shell=True) call('rm brute*.log;', shell=True) ensure_wdir(project='brute') empty_queues(project='brute') # start 80 workers on 10 PBS nodes (8 on each) for node in range(1, 11, 1): pbsjob = '''# Shell for the job: #PBS -S /bin/bash # request 1 node, 8 cores #PBS -lnodes=1:cores8 # job requires at most n hours wallclock time #PBS -lwalltime=08:00:00 cd /home/nicolas/brute fjd-recruiter --project brute hire 8 python -c "import time; time.sleep(16*60*60)" # keep PBS job alive ''' with open("pbsjobs/pbsjob{}".format(node), 'w') as f: