def run_example(self):
ensure_wdir(project=self.project)
empty_queues(project=self.project)
os.chdir(self.ex_dir)
if os.path.exists('logfiles'):
rmtree('logfiles')
os.mkdir('logfiles')
call('./create_jobs.py {}'.format(self.project), shell=True)
call('fjd-recruiter --project {} hire {}'\
.format(self.project, self.num_workers), shell=True)
call('fjd-dispatcher --project {} --end_when_jobs_are_done'\
.format(self.project), shell=True)
def __init__(self, exe, repeat=1, parameters=[], project=None, num_workers=0,
callback=None, curdir=''):
if not exe or exe == '':
print('[fjd] Please specify an executable command (--exe).')
sys.exit(2)
if repeat > 1 and len(parameters) > 0:
print('[fjd] Only one of --repeat and --parameters can be set at a time.')
sys.exit(2)
empty_queues(project=project)
self.wdir = ensure_wdir(project)
if len(parameters) > 1:
for i, p in enumerate(parameters):
job = '{}/jobqueue/job{}'.format(self.wdir, i)
with open(job, 'w') as f:
f.write('#!/bin/bash\n')
cur_exe = exe
ext_params = []
for j, param in enumerate(str(p).split('#')):
if '${}'.format(j+1) in cur_exe:
cur_exe = cur_exe.replace('${}'.format(j+1), str(param))
else:
ext_params.append(str(p))
f.write('{exe} {params}'.format(exe=cur_exe, params=' '.join(ext_params)))
os.chmod(job, 0o777)
else:
for i in range(repeat):
job = '{}/jobqueue/job{}'.format(self.wdir, i)
with open(job, 'w') as f:
f.write('#!/bin/bash\n')
f.write(exe)
os.chmod(job, 0o777)
if num_workers == 0:
num_workers = cpu_count() - 1
num_workers = min(num_workers, cpu_count())
recruiter = Recruiter(num_workers=num_workers, project=project,
curdir=curdir)
recruiter.hire()
Dispatcher(project=project, callback=callback)
'''
import sys
import os
import itertools
import numpy as np
import random
from subprocess import call
from fjd import Dispatcher
from fjd.utils import ensure_wdir, empty_queues
# clean up
call('rm pbsjobs/pbsjob*', shell=True)
call('rm brute*.log;', shell=True)
ensure_wdir(project='brute')
empty_queues(project='brute')
# start 80 workers on 10 PBS nodes (8 on each)
for node in range(1, 11, 1):
pbsjob = '''# Shell for the job:
#PBS -S /bin/bash
# request 1 node, 8 cores
#PBS -lnodes=1:cores8
# job requires at most n hours wallclock time
#PBS -lwalltime=08:00:00
cd /home/nicolas/brute
fjd-recruiter --project brute hire 8
python -c "import time; time.sleep(16*60*60)" # keep PBS job alive
'''
with open("pbsjobs/pbsjob{}".format(node), 'w') as f:
