def test_j_Queues(self):
from Ganga.GPI import queues, Job, GenericSplitter
queues
# -- QUEUES EXAMPLE START
for i in range(1, 10):
j = Job()
queues.add(j.submit)
# -- QUEUES EXAMPLE STOP
# -- QUEUES FUNCTION START
def f(x):
print x
queues.add(f, args=(123,))
# -- QUEUES FUNCTION STOP
# -- QUEUES SPLIT START
j = Job()
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [i for i in range(0, 10)]
j.parallel_submit = True
j.submit()
def submit_job(brunel_app, reco_type, input_files=None, local=RUN_LOCAL):
# Set EvtMax depending on if this is a local job
brunel_app.extraOpts += 'from Configurables import Brunel\n'
brunel_app.extraOpts += 'Brunel().EvtMax = {}'.format(2 * int(local) - 1)
# Configure the corresponding Job
job = Job(name='VP hybrid distortions',
comment='{reco_type} reconstruction {suffix}'.format(
reco_type=reco_type, suffix=['', '(local)'][local]),
application=brunel_app,
splitter=SplitByFiles(filesPerJob=1, ignoremissing=True),
parallel_submit=True)
if local:
job.backend = Local()
job.outputfiles = [LocalFile('*.xdst'), LocalFile('*.root')]
job.inputdata = dataset[:1]
else:
job.backend = Dirac()
job.outputfiles = [DiracFile('*.xdst'), DiracFile('*.root')]
job.inputdata = dataset
job.inputfiles = input_files or []
queues.add(job.submit)
return True
def diracAPI_interactive(connection_attempts=5):
'''
Run an interactive server within the DIRAC environment.
'''
import os
import sys
import time
import inspect
import traceback
from GangaDirac.Lib.Server.InspectionClient import runClient
serverpath = os.path.join(
os.path.dirname(inspect.getsourcefile(runClient)),
'InspectionServer.py')
queues.add(
execute("execfile('%s')" % serverpath, timeout=None, shell=False))
#time.sleep(1)
sys.stdout.write(
"\nType 'q' or 'Q' or 'exit' or 'exit()' to quit but NOT ctrl-D")
i = 0
excpt = None
while i < connection_attempts:
try:
runClient()
break
except:
if i == (connection_attempts - 1):
excpt = traceback.format_exc()
finally:
i += 1
return excpt
def testSetParentOnLoad(self):
"""
Test that the parents are set correctly on load
"""
from Ganga.GPI import jobs, queues, Executable, Local
from Ganga.GPIDev.Base.Proxy import isType
def flush_full_job():
mj = jobs(0)
mj.comment = "Make sure I'm dirty " + ''.join(random.choice(string.ascii_uppercase) for _ in range(5))
mj._impl._getRegistry()._flush([j])
# Make sure the main job is fully loaded
j = jobs(0)
assert isType(j.application, Executable)
assert isType(j.backend, Local)
assert j.application.exe == "sleep"
# fire off a load of threads to flush
for i in range(0, 20):
queues.add(flush_full_job)
# Now loop over and force the load of all the subjobs
for sj in j.subjobs:
assert sj.splitter is None
assert isType(sj.application, Executable)
assert isType(sj.backend, Local)
assert sj.application.exe == "sleep"
assert sj.application.args == ['400']
assert sj._impl._getRoot() is j._impl
def diracAPI_interactive(connection_attempts=5):
'''
Run an interactive server within the DIRAC environment.
'''
import os
import sys
import time
import inspect
import traceback
from GangaDirac.Lib.Server.InspectionClient import runClient
serverpath = os.path.join(os.path.dirname(inspect.getsourcefile(runClient)), 'InspectionServer.py')
queues.add(execute("execfile('%s')" % serverpath, timeout=None, shell=False))
#time.sleep(1)
sys.stdout.write( "\nType 'q' or 'Q' or 'exit' or 'exit()' to quit but NOT ctrl-D")
i = 0
excpt = None
while i < connection_attempts:
try:
runClient()
break
except:
if i == (connection_attempts - 1):
excpt = traceback.format_exc()
finally:
i += 1
return excpt
def submit(self, **opts): # called on client, so job_info is Job object
"""Log submit event on client."""
# if this job has a master and it is the first subjob then sent
# submitted for master job
if self.job_info.master is not None:
if self.job_info.id == 0:
masterjob_msg = self.getMessage('submitted')
masterjob_msg['subjobs'] = len(self.job_info.master.subjobs)
masterjob_msg['ganga_job_id'] = str(
masterjob_msg['ganga_job_id']).split('.')[0]
# override ganga_job_uuid as the message 'from the master' is
# really sent from the subjob
masterjob_msg['ganga_job_uuid'] = masterjob_msg[
'ganga_master_uuid']
masterjob_msg['ganga_master_uuid'] = 0
self.send(masterjob_msg)
from Ganga.Utility import Config
gangausername = Config.getConfig('Configuration')['user']
self.job_info.info.monitoring_links.append(
('http://gangamon.cern.ch/ganga/#user=%s' % gangausername, 'dashboard'))
# send submitted for this job
msg = self.getMessage('submitted')
from Ganga.GPI import queues
queues.add(self.send, (msg))
def testSetParentOnLoad(self):
"""
Test that the parents are set correctly on load
"""
from Ganga.GPI import jobs, queues, Executable, Local
from Ganga.GPIDev.Base.Proxy import isType, stripProxy
def flush_full_job():
mj = jobs(0)
mj.comment = "Make sure I'm dirty " + ''.join(
random.choice(string.ascii_uppercase) for _ in range(5))
stripProxy(mj)._getRegistry()._flush([stripProxy(mj)])
# Make sure the main job is fully loaded
j = jobs(0)
assert isType(j.application, Executable)
assert isType(j.backend, Local)
assert j.application.exe == "sleep"
# fire off a load of threads to flush
for i in range(0, 20):
queues.add(flush_full_job)
# Now loop over and force the load of all the subjobs
for sj in j.subjobs:
assert sj.splitter is None
assert isType(sj.application, Executable)
assert isType(sj.backend, Local)
assert sj.application.exe == "sleep"
assert sj.application.args == ['400']
assert stripProxy(sj)._getRoot() is stripProxy(j)
assert stripProxy(sj.application)._getRoot() is stripProxy(j)
def test_j_Queues(self):
from Ganga.GPI import queues, Job, GenericSplitter
queues
# -- QUEUES EXAMPLE START
for i in range(1, 10):
j = Job()
queues.add(j.submit)
# -- QUEUES EXAMPLE STOP
# -- QUEUES FUNCTION START
def f(x):
print x
queues.add(f, args=(123, ))
# -- QUEUES FUNCTION STOP
# -- QUEUES SPLIT START
j = Job()
j.splitter = GenericSplitter()
j.splitter.attribute = 'application.args'
j.splitter.values = [i for i in range(0, 10)]
j.parallel_submit = True
j.submit()
def queueDownload(joblist,targetDirBase=None,force_redownload=False):
'''joblist may be jobs.select, or array of jobs or array of jobids'''
for job in joblist:
job,joblist = getJobList(job)
targetdir=None
if isinstance(targetDirBase,str):
targetdir = targetDirBase+"/"+job.name
queues.add(download,kwargs={"job":job,"targetDir":targetdir,"force_redownload":force_redownload})
def stop(self, exitcode, **opts):
"""Log stop event on worker node."""
if exitcode == 0:
event = 'finished'
else:
event = 'failed'
message = self.getMessage(event)
from Ganga.GPI import queues
queues.add(self.send, (message))
def _get_publisher(server, port, username, password):
# FIXME: this assumes server/port/username/password cannot change and
# caches a singleton publisher
global _publisher
if _publisher is None:
_publisher = MSGUtil.createPublisher(server, port, username, password)
from Ganga.GPI import queues
queues.add(_publisher.start)
return _publisher
def queueDownload(joblist, targetDirBase=None, force_redownload=False):
'''joblist may be jobs.select, or array of jobs or array of jobids'''
for job in joblist:
job, joblist = getJobList(job)
targetdir = None
if isinstance(targetDirBase, str):
targetdir = targetDirBase + "/" + job.name
queues.add(download,
kwargs={
"job": job,
"targetDir": targetdir,
"force_redownload": force_redownload
})
def send(self, message):
"""Send the message to the configured destination."""
# get publisher
p = _get_publisher(
self.config_info['server'],
self.config_info['port'],
self.config_info['username'],
self.config_info['password'],
)
# send message
headers = {'persistent': 'true'}
from Ganga.GPI import queues
queues.add(
p.send, (self.config_info['message_destination'], repr(message), headers))
def test_c_QueueSubmits(self):
from Ganga.GPI import jobs, queues
for j in jobs:
print('adding job', j.id, 'to queue for submission')
queues.add(j.submit)
while queues.totalNumUserThreads() > 0:
print('remaining threads:', queues.totalNumUserThreads())
time.sleep(1)
print('remaining threads:', queues.totalNumUserThreads())
# All user threads should have terminated by now
for j in jobs:
print('checking job', j.id)
assert j.status != 'new'
def test_c_QueueSubmits(self):
from Ganga.GPI import jobs, queues
for j in jobs:
print('adding job', j.id, 'to queue for submission')
queues.add(j.submit)
while queues.totalNumUserThreads() > 0:
print('remaining threads:', queues.totalNumUserThreads())
time.sleep(1)
print('remaining threads:', queues.totalNumUserThreads())
# All user threads should have terminated by now
for j in jobs:
print('checking job', j.id)
assert j.status != 'new'
def fixMyJobs(joblist):
queueresubmission(joblist)
for j in joblist:
queues.add(resubmitStrangeJobs,kwargs={"j":j})
def queueresubmission(joblist):
for j in joblist:
queues.add(resub,kwargs={"j":j})
def _auto_remove(self):
"""
Remove called when job is removed as long as config option allows
"""
if self.lfn != '':
queues.add(self.remove)
def diracAPI_async(cmd, timeout=120):
'''
Execute DIRAC API commands from w/in Ganga.
'''
return queues.add(execute(cmd, timeout=timeout))
def _auto_remove(self):
"""
Remove called when job is removed as long as config option allows
"""
if self.lfn != '':
queues.add(self.remove)
def start(self, **opts):
"""Log start event on worker node."""
message = self.getMessage('running')
from Ganga.GPI import queues
queues.add(self.send, (message))
def queueresubmission(joblist):
for j in joblist:
queues.add(resub, kwargs={"j": j})
def fixMyJobs(joblist):
queueresubmission(joblist)
for j in joblist:
queues.add(resubmitStrangeJobs, kwargs={"j": j})
def diracAPI_async(cmd, timeout=120):
'''
Execute DIRAC API commands from w/in Ganga.
'''
return queues.add(execute(cmd, timeout=timeout))
def GangaDiracSplitter(inputs, filesPerJob, maxFiles, ignoremissing):
"""
Generator that yields a datasets for dirac split jobs
"""
split_files = []
i = inputs.__class__()
if len(inputs.getLFNs()) != len(inputs.files):
raise SplittingError(
"Error trying to split dataset using DIRAC backend with non-DiracFile in the inputdata")
file_replicas = {}
from Ganga.GPI import queues
for i in inputs:
#logging.debug( "getting metadata: %s" % str(i.lfn) )
queues.add(i.getReplicas)
logger.info("Requesting LFN replica info")
# This finds all replicas for all LFNs...
# This will probably struggle for LFNs which don't exist
all_lfns = [i.locations for i in inputs]
while [] in all_lfns:
import time
time.sleep(0.5)
all_lfns = [i.locations for i in inputs]
logger.info("Got replicas")
for i in inputs:
file_replicas[i.lfn] = i.locations
#logger.info( "%s" % str( i.accessURL() ) )
logger.debug("found all replicas")
super_dict = dict()
for lfn, repz in file_replicas.iteritems():
sitez = set([])
for i in repz:
# print i
sitez.add(i)
super_dict[lfn] = sitez
allSubSets = []
allChosenSets = {}
logger.info("Determining overlap")
import random
for i in super_dict.keys():
# Randomly Select 2 SE as the starting point for spliting jobs
if len(super_dict[i]) > 2:
req_sitez = set([])
chosen = random.sample(super_dict[i], 2)
for s in chosen:
req_sitez.add(s)
# Keep the 2 or less SE as the SE of choice
else:
req_sitez = set([])
for s in super_dict[i]:
req_sitez.add(s)
allChosenSets[i] = req_sitez
logger.debug("Found all SE in use")
Tier1Sites = set([])
for i in super_dict.keys():
req_sitez = allChosenSets[i]
_this_subset = []
# Starting with i, populate subset with LFNs which have an
# overlap of at least 2 SE
for k in super_dict.keys():
if req_sitez.issubset(super_dict[k]):
if len(_this_subset) >= filesPerJob:
break
_this_subset.append(str(k))
super_dict.pop(k)
if len(_this_subset) > 0:
allSubSets.append(_this_subset)
split_files = allSubSets
logger.info("Created %s subsets" % str(len(split_files)))
#logger.info( "Split Files: %s" % str(split_files) )
for dataset in split_files:
yield dataset
