def __init__(self):
Submitter.__init__(self)
self.parser.description = 'Submit worker agents to LCG. '+self.parser.description
self.parser.add_option('--delay',dest='delay',type='int',default=0,help='delay in seconds in between the worker submission (to avoid spikes in the worker agent registration)')
self.parser.add_option('--CE',dest='CE',type='string',default='',help='set LCG.CE attribute, example: ce.cyf-kr.edu.pl:2119/jobmanager-pbs-gear, tbn20.nikhef.nl:2119/jobmanager-pbs-qlong')
self.parser.add_option('--CE-list',dest='CE_list',type='string',default='',help='a file containing the list of CEs to be used; for each job the LCG.CE attribute will be assigned using round robin algorithm')
self.download=True
#the number of the current CE to be used
self.current = 0
#Computing Element list
self.CEs = []
def submit_worker(self,j,ce=None):
j.backend = Ganga.GPI.LCG()
#j.backend = Ganga.GPI.Local()
if self.CEs != []:
#set the CE for a given job
CE = self.CEs[self.current]
print 'Select CE:', CE
j.backend.CE = CE
#update and normalize last CE number
self.current = (self.current + 1) % len(self.CEs)
elif ce is not None:
j.backend.CE = ce #only if the CEs list is empty
#call the superclass' method to set other diane-specific job settings
Submitter.submit_worker(self,j)
if self.options.delay:
print 'now sleeping for %d seconds '%self.options.delay
time.sleep(self.options.delay)
def initialize(self):
Submitter.initialize(self)
self.CEs = []
if self.options.CE_list != '' and self.options.CE != '':
print 'WARNING: both --CE and --CE-list options are specified. Use one or the other. The script will now exit...'
sys.exit(0)
if self.options.CE_list != '':
f = open(self.options.CE_list)
try:
for line in f:
self.CEs.append(line.strip())
finally:
f.close()
if self.options.CE != '':
self.CEs = [self.options.CE]
if self.options.delay<0:
self.options.delay = 0
#!/usr/bin/env ganga
#-*-python-*-
if __name__ == '__main__':
from diane.submitters import Submitter
prog = Submitter()
prog.download=False
prog.parser.description = "Submit worker agents to LSF batch system. "+prog.parser.description
prog.parser.add_option("--delay",dest="delay",type="int",default=0,help="delay in seconds in between the worker submission (to avoid spikes in the worker agent registration)")
prog.parser.add_option("--queue",dest="queue",type="string",default='',help="set LSF queue")
prog.parser.add_option("--extraopts",dest="extraopts",type="string",default='',help="Pass extra options to batch system Actual options depends on batch system. See GANGA Batch interface for more information.")
prog.initialize()
import time
for i in range(prog.options.N_WORKERS):
j = Job()
j.backend=LSF(queue=prog.options.queue)
if prog.options.extraopts:
j.backend.extraopts = prog.options.extraopts
prog.submit_worker(j)
if prog.options.delay:
print "now sleeping for %d seconds "%prog.options.delay
time.sleep(prog.options.delay)
#!/usr/bin/env ganga
#-*-python-*-
if __name__ == '__main__':
from diane.submitters import Submitter
prog = Submitter()
prog.parser.description = "Submit worker agents locally. "+prog.parser.description
prog.initialize()
for i in range(prog.options.N_WORKERS):
j = Job()
j.backend=Local()
prog.submit_worker(j)
#
# This script may be placed in ~/diane/submitters (it will then take precedence over any submitters with the same name contained in the release tree).
#
# usage:
# diane-submitter SAGA [options]
#
# print all available options: diane-submitter SAGA -h
#
import sys
from diane import getLogger
logger=getLogger('SAGASubmitter')
from diane.submitters import Submitter
prog = Submitter()
prog.download=False
prog.parser.description="Submit worker agents using SAGA. "+prog.parser.description
# These parameters are required for the SAGA backend
prog.parser.add_option("--jobservice-url",type="string",default="",help="the remote job service url (e.g. gram://qb1.loni.org/jobmanager-pbs)")
prog.parser.add_option("--filesystem-url",type="string",default="",help="the remote filesystem root (e.g. gsiftp://qb1.loni.org/work/oweidner/diane-worker/)")
prog.parser.add_option("--allocation",type="string",default="",help="the allocation to be used for job accounting")
prog.parser.add_option("--delay",dest="delay",type="int",default=0,help="delay in seconds in between the worker submission (to avoid spikes in the worker agent registration)")
# this wrapper will start a number of worker agents using ssh hosts specified by $PBS_NODEFILE
# it assumes, however, that all the nodes are connected by a shared file system
sh_download_multinode_wrapper = '''#!/bin/sh
#!/usr/bin/env ganga
#-*-python-*-
if __name__ == '__main__':
from diane.submitters import Submitter
prog = Submitter()
prog.parser.add_option('--node-number',type="int",default=1,help="Use a number of worker nodes to run one worker agent on each.")
sh_download_multinode_wrapper = '''#!/bin/sh
#wget %(DIANE_DOWNLOAD_URL)s/diane-install
#python ./diane-install --download-url=%(DIANE_DOWNLOAD_URL)s --prefix=$PWD/diane %(DIANE_VERSION)s
#BINDIR=$PWD/diane/install/%(DIANE_VERSION)s/bin
BINDIR=%(DIANE_TOP)s/bin
wids=''
for node in ###NODES###;
do
pwd
# create a workdir
wdir=`mktemp -d ${PWD}/tmp.workdir.XXXXXXX`
echo 'spawning worker to ' $node ' workdir ' $wdir
# copy the required input files to the subdirectory first
# TODO: having hardcoded file names here is suboptimal, in the future this list should be created automatically
