def __init__(self): Submitter.__init__(self) self.parser.description = 'Submit worker agents to LCG. '+self.parser.description self.parser.add_option('--delay',dest='delay',type='int',default=0,help='delay in seconds in between the worker submission (to avoid spikes in the worker agent registration)') self.parser.add_option('--CE',dest='CE',type='string',default='',help='set LCG.CE attribute, example: ce.cyf-kr.edu.pl:2119/jobmanager-pbs-gear, tbn20.nikhef.nl:2119/jobmanager-pbs-qlong') self.parser.add_option('--CE-list',dest='CE_list',type='string',default='',help='a file containing the list of CEs to be used; for each job the LCG.CE attribute will be assigned using round robin algorithm') self.download=True #the number of the current CE to be used self.current = 0 #Computing Element list self.CEs = []
def submit_worker(self,j,ce=None): j.backend = Ganga.GPI.LCG() #j.backend = Ganga.GPI.Local() if self.CEs != []: #set the CE for a given job CE = self.CEs[self.current] print 'Select CE:', CE j.backend.CE = CE #update and normalize last CE number self.current = (self.current + 1) % len(self.CEs) elif ce is not None: j.backend.CE = ce #only if the CEs list is empty #call the superclass' method to set other diane-specific job settings Submitter.submit_worker(self,j) if self.options.delay: print 'now sleeping for %d seconds '%self.options.delay time.sleep(self.options.delay)
def initialize(self): Submitter.initialize(self) self.CEs = [] if self.options.CE_list != '' and self.options.CE != '': print 'WARNING: both --CE and --CE-list options are specified. Use one or the other. The script will now exit...' sys.exit(0) if self.options.CE_list != '': f = open(self.options.CE_list) try: for line in f: self.CEs.append(line.strip()) finally: f.close() if self.options.CE != '': self.CEs = [self.options.CE] if self.options.delay<0: self.options.delay = 0
#!/usr/bin/env ganga #-*-python-*- if __name__ == '__main__': from diane.submitters import Submitter prog = Submitter() prog.download=False prog.parser.description = "Submit worker agents to LSF batch system. "+prog.parser.description prog.parser.add_option("--delay",dest="delay",type="int",default=0,help="delay in seconds in between the worker submission (to avoid spikes in the worker agent registration)") prog.parser.add_option("--queue",dest="queue",type="string",default='',help="set LSF queue") prog.parser.add_option("--extraopts",dest="extraopts",type="string",default='',help="Pass extra options to batch system Actual options depends on batch system. See GANGA Batch interface for more information.") prog.initialize() import time for i in range(prog.options.N_WORKERS): j = Job() j.backend=LSF(queue=prog.options.queue) if prog.options.extraopts: j.backend.extraopts = prog.options.extraopts prog.submit_worker(j) if prog.options.delay: print "now sleeping for %d seconds "%prog.options.delay time.sleep(prog.options.delay)
#!/usr/bin/env ganga #-*-python-*- if __name__ == '__main__': from diane.submitters import Submitter prog = Submitter() prog.parser.description = "Submit worker agents locally. "+prog.parser.description prog.initialize() for i in range(prog.options.N_WORKERS): j = Job() j.backend=Local() prog.submit_worker(j)
# # This script may be placed in ~/diane/submitters (it will then take precedence over any submitters with the same name contained in the release tree). # # usage: # diane-submitter SAGA [options] # # print all available options: diane-submitter SAGA -h # import sys from diane import getLogger logger=getLogger('SAGASubmitter') from diane.submitters import Submitter prog = Submitter() prog.download=False prog.parser.description="Submit worker agents using SAGA. "+prog.parser.description # These parameters are required for the SAGA backend prog.parser.add_option("--jobservice-url",type="string",default="",help="the remote job service url (e.g. gram://qb1.loni.org/jobmanager-pbs)") prog.parser.add_option("--filesystem-url",type="string",default="",help="the remote filesystem root (e.g. gsiftp://qb1.loni.org/work/oweidner/diane-worker/)") prog.parser.add_option("--allocation",type="string",default="",help="the allocation to be used for job accounting") prog.parser.add_option("--delay",dest="delay",type="int",default=0,help="delay in seconds in between the worker submission (to avoid spikes in the worker agent registration)") # this wrapper will start a number of worker agents using ssh hosts specified by $PBS_NODEFILE # it assumes, however, that all the nodes are connected by a shared file system sh_download_multinode_wrapper = '''#!/bin/sh
#!/usr/bin/env ganga #-*-python-*- if __name__ == '__main__': from diane.submitters import Submitter prog = Submitter() prog.parser.add_option('--node-number',type="int",default=1,help="Use a number of worker nodes to run one worker agent on each.") sh_download_multinode_wrapper = '''#!/bin/sh #wget %(DIANE_DOWNLOAD_URL)s/diane-install #python ./diane-install --download-url=%(DIANE_DOWNLOAD_URL)s --prefix=$PWD/diane %(DIANE_VERSION)s #BINDIR=$PWD/diane/install/%(DIANE_VERSION)s/bin BINDIR=%(DIANE_TOP)s/bin wids='' for node in ###NODES###; do pwd # create a workdir wdir=`mktemp -d ${PWD}/tmp.workdir.XXXXXXX` echo 'spawning worker to ' $node ' workdir ' $wdir # copy the required input files to the subdirectory first # TODO: having hardcoded file names here is suboptimal, in the future this list should be created automatically