ifspec.dataset = jspec.prodDBlock
ifspec.prodDBlock = jspec.prodDBlock
ifspec.type = 'input'
jspec.addFile(ifspec)
if app.input_type:
itype = app.input_type
else:
itype = m.group(5)
if jspec.transformation.endswith(
"_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(
job.inputdata.names))
else:
jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(
job.inputdata.names))
# Log files.
lfspec = FileSpec()
lfspec.lfn = '%s.job.log.tgz' % jspec.jobName
lfspec.destinationDBlock = jspec.destinationDBlock
lfspec.destinationSE = jspec.destinationSE
lfspec.dataset = jspec.destinationDBlock
lfspec.type = 'log'
jspec.addFile(lfspec)
return jspec
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('ProdTrans', 'Panda', ProdTransPandaRTHandler)
class RootLCGRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
runScript, inputsandbox, rootenv = downloadWrapper(app)
# propage command line args
argList = [str(s) for s in app.args]
return LCGJobConfig(runScript, inputsandbox, argList,
app._getParent().outputsandbox, rootenv)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Root', 'Cronus', RootRTHandler)
allHandlers.add('Root', 'LSF', RootLocalRTHandler)
allHandlers.add('Root', 'Local', RootLocalRTHandler)
allHandlers.add('Root', 'Interactive', RootRTHandler)
# Easy way to test the system with ROOT dowloaded on demand
#allHandlers.add('Root','Local', RootDownloadHandler)
allHandlers.add('Root', 'PBS', RootLocalRTHandler)
allHandlers.add('Root', 'SGE', RootLocalRTHandler)
allHandlers.add('Root', 'Condor', RootRTHandler)
allHandlers.add('Root', 'LCG', RootLCGRTHandler)
allHandlers.add('Root', 'CREAM', RootLCGRTHandler)
allHandlers.add('Root', 'ARC', RootLCGRTHandler)
allHandlers.add('Root', 'TestSubmitter', RootRTHandler)
allHandlers.add('Root', 'Remote', RootLCGRTHandler)
Return value: None
"""
# job = self.getJobObject()
# topdir = self.workdir.rstrip( os.sep )
# path = os.path.join( topdir, filename ).rstrip( os.sep )
# job.viewFile( path = path, command = command )
return None
# def remove_workdir(self):
# if config['remove_workdir']:
# import shutil
# try:
# shutil.rmtree(self.workdir)
# except OSError as x:
# logger.warning('problem removing the workdir %s: %s',str(self.id),str(x))
##
##############################################################################
##############################################################################
##
## Register a default runtime handler
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.Lib.Executable import RTHandler
allHandlers.add("Executable", "SAGA", RTHandler)
COMMAND='/usr/bin/env python %s %s' %
(os.path.basename(app.script.name), '%s'),
JOINER=' ',
#INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'')
)
else:
python_wrapper = script_generator(
python_wrapper,
remove_unreplaced=False,
FIXARGS='script_args=[formatVar(v) for v in script_args]',
COMMAND=
'export DISPLAY=\"localhoast:0.0\" && root -l -q \"%s(%s)\"' %
(os.path.basename(app.script.name), '%s'),
JOINER=',',
#INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'')
)
f.write(python_wrapper)
f.close()
dirac_script = script_generator(lhcbdiracAPI_script_template(),
**params)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Root', 'Dirac', LHCbRootDiracRunTimeHandler)
return rc
# Main
if __name__ == '__main__':
err = None
try:
rc = run_Im3ShapeApp()
except Exception as x:
rc = -9999
print('Exception occured in running app.')
print('Err was: ' + str(x))
print("files on WN: " + str(listdir('.')))
print("environ: %s" % environ)
raise
print("files on WN: " + str(listdir('.')))
###OUTPUTFILESINJECTEDCODE###
exit(rc)
"""
return script_template
# \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add("Im3ShapeApp", "Dirac", Im3ShapeDiracRTHandler)
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
# else assume it's a system binary, so we don't need to
# transport anything to the sharedir
else:
prepared_exe = app.exe
elif isinstance(app.exe, File):
logger.info("Submitting a prepared application; taking any input files from %s" % (
app.is_prepared.name))
prepared_exe = File(os.path.join(os.path.join(
shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name)))
return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Executable', 'LSF', RTHandler)
allHandlers.add('Executable', 'Local', RTHandler)
allHandlers.add('Executable', 'PBS', RTHandler)
allHandlers.add('Executable', 'SGE', RTHandler)
allHandlers.add('Executable', 'Condor', RTHandler)
allHandlers.add('Executable', 'LCG', LCGRTHandler)
allHandlers.add('Executable', 'gLite', gLiteRTHandler)
allHandlers.add('Executable', 'TestSubmitter', RTHandler)
allHandlers.add('Executable', 'Interactive', RTHandler)
allHandlers.add('Executable', 'Batch', RTHandler)
allHandlers.add('Executable', 'Cronus', RTHandler)
allHandlers.add('Executable', 'Remote', LCGRTHandler)
allHandlers.add('Executable', 'CREAM', LCGRTHandler)
allHandlers.add('Executable', 'ARC', LCGRTHandler)
filename : name of file to be viewed
=> Path specified relative to work directory
command : command to be used for file viewing
Return value: None
"""
#job = self.getJobObject()
#topdir = self.workdir.rstrip( os.sep )
#path = os.path.join( topdir, filename ).rstrip( os.sep )
#job.viewFile( path = path, command = command )
return None
# def remove_workdir(self):
# if config['remove_workdir']:
# import shutil
# try:
# shutil.rmtree(self.workdir)
# except OSError as x:
# logger.warning('problem removing the workdir %s: %s',str(self.id),str(x))
##
##############################################################################
##############################################################################
##
## Register a default runtime handler
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.Lib.Executable import RTHandler
allHandlers.add('Executable', 'SAGA', RTHandler)
# Generate the script which is to be executed for us on the WN
scriptToRun = generateWNScript(job_command, app)
input_sand.append(scriptToRun)
logger.debug("input_sand: %s" % input_sand)
# It's this authors opinion that the script should be in the PATH on the WN
# As it stands policy is that is isn't so we have to call it in a relative way, hence "./"
c = StandardJobConfig(
'./' + os.path.join(scriptToRun.subdir, scriptToRun.name),
input_sand, [], output_sand)
return c
allHandlers.add('GaudiExec', 'Local', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'Condor', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'Interactive', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'Batch', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'LSF', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'PBS', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'SGE', GaudiExecRTHandler)
def generateDiracInput(app):
"""
Construct a DIRAC input which does not need to be unique to each job but is required to have a unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
"""
from pandatools import Client
from pandatools import AthenaUtils
from taskbuffer.JobSpec import JobSpec
from taskbuffer.FileSpec import FileSpec
job = app._getParent()
masterjob = job._getRoot()
logger.debug('AthenaJediRTHandler prepare called for %s',
job.getFQID('.'))
# in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter
return {}
from Ganga.GPIDev.Credentials import GridProxy
gridProxy = GridProxy()
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
for app in ['Athena', 'ExecutableDQ2', 'RootDQ2']:
allHandlers.add(app, 'Jedi', AthenaJediRTHandler)
from Ganga.Utility.Config import getConfig, ConfigError
configDQ2 = getConfig('DQ2')
configJedi = getConfig('Jedi')
from Ganga.Utility.logging import getLogger
logger = getLogger()
return c
class LCGRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
c = LCGJobConfig(app.exe, app.inputs, app.args, app.outputs, app.envs)
c.monitoring_svc = mc['GangaTutorial']
return c
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('PrimeFactorizer', 'LSF', RTHandler)
allHandlers.add('PrimeFactorizer', 'Local', RTHandler)
allHandlers.add('PrimeFactorizer', 'PBS', RTHandler)
allHandlers.add('PrimeFactorizer', 'SGE', RTHandler)
allHandlers.add('PrimeFactorizer', 'Condor', RTHandler)
allHandlers.add('PrimeFactorizer', 'LCG', LCGRTHandler)
allHandlers.add('PrimeFactorizer', 'TestSubmitter', RTHandler)
allHandlers.add('PrimeFactorizer', 'Interactive', RTHandler)
allHandlers.add('PrimeFactorizer', 'Batch', RTHandler)
##############################################################
## handler for NG
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.GPIDev.Adapters.IRuntimeHandler import IRuntimeHandler
# from Ganga.GPIDev.Base.Filters import allComponentFilters
# v = allComponentFilters['files'](v,self._schema.getItem('file_or_files'))
# if not type(v) is type([]):
# v = [v]
# return v
# # looks like in GPI we see everything double ;-)
# def _attribute_filter__get__(self,n,v):
# return v*2
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.Lib.Executable.Executable import RTHandler
class TestRTHandler(RTHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
if appconfig.find('error') != -1:
x = 'triggered failure during config'
raise Exception(x)
return RTHandler.prepare(self, app, appconfig, appmasterconfig,
jobmasterconfig)
allHandlers.add('TestApplication', 'TestSubmitter', TestRTHandler)
## FIXME: Test for advanced properties...
## j = Job()
## a = TestAdvancedProperties()
## j.application = a
File(
os.path.join(
os.path.dirname(inspect.getsourcefile(GaudiPython)),
'options/GaudiPythonExample.py'))
]
else:
for f in self.script:
f.name = fullpath(f.name)
return
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
# Associate the correct run-time handlers to GaudiPython for various backends.
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler
from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler
for backend in [
'LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote'
]:
allHandlers.add('GaudiPython', backend, LHCbGaudiRunTimeHandler)
allHandlers.add('GaudiPython', 'Dirac', LHCbGaudiDiracRunTimeHandler)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
for i,rm in enumerate(self.ratMacro):
j = self.createSubjob(job)
j.application.ratMacro = rm
if self.outputFile!=[]:
j.application.outputFile = self.outputFile[i]
if self.inputFile!=[]:
j.application.inputFile = self.inputFile[i]
if self.nEvents!=[]:
j.application.nEvents = self.nEvents[i]
subjobs.append(j)
return subjobs
###################################################################
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaSNOplus.Lib.RTHandlers.RTRATUser import UserRTHandler, UserLCGRTHandler, UserWGRTHandler, UserDiracRTHandler
allHandlers.add('RATUser','Local', UserRTHandler)
allHandlers.add('RATUser','PBS', UserRTHandler)
allHandlers.add('RATUser','SGE', UserRTHandler)
allHandlers.add('RATUser','Condor', UserRTHandler)
allHandlers.add('RATUser','LCG', UserLCGRTHandler)
allHandlers.add('RATUser','Dirac', UserDiracRTHandler)
allHandlers.add('RATUser','TestSubmitter', UserRTHandler)
allHandlers.add('RATUser','Interactive', UserRTHandler)
allHandlers.add('RATUser','Batch', UserRTHandler)
allHandlers.add('RATUser','WestGrid', UserWGRTHandler)
logger = Ganga.Utility.logging.getLogger()
uploadSources(os.path.dirname(self.inputsandbox),os.path.basename(self.inputsandbox))
if not self.inputsandbox == tmp_user_area_name:
logger.info('Removing source tarball %s ...' % self.inputsandbox )
os.remove(self.inputsandbox)
return taskParamMap
def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig):
'''prepare the subjob specific configuration'''
job = app._getParent()
logger.debug('AthenaJediRTHandler prepare called for %s', job.getFQID('.'))
# in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter
return {}
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
for app in ['Athena', 'ExecutableDQ2', 'RootDQ2']:
allHandlers.add(app,'Jedi',AthenaJediRTHandler)
from Ganga.Utility.Config import getConfig, ConfigError
configDQ2 = getConfig('DQ2')
configJedi = getConfig('Jedi')
from Ganga.Utility.logging import getLogger
logger = getLogger()
path = tempfile.mkdtemp(prefix='GANGA_')
logger.info(
'Bender application will be prepared in the temporary directory %s'
% path)
the_path = prepare_cmake_app(project, version, path)
#
scripts = kwarsg.pop('scripts', [])
if script: scripts.append(script)
#
from Ganga.GPI import OstapRun as _OR ## black magic
return _OR(scripts=scripts, directory=the_path, **kwargs)
## export it!
from Ganga.Runtime.GPIexport import exportToGPI
exportToGPI('prepareBender', prepareBender, 'Functions')
exportToGPI('prepareBenderRun', prepareBenderRun, 'Functions')
exportToGPI('prepareOstapRun', prepareOstapRun, 'Functions')
##
for app in ('BenderModule', 'BenderRun', 'OstapRun'):
allHandlers.add(app, 'Dirac', GaudiExecDiracRTHandler)
for backend in ("Local", "Interactive", "LSF", "PBS", "SGE", "Condor"):
allHandlers.add(app, backend, GaudiExecRTHandler)
# =============================================================================
# The END
# =============================================================================
def addLocalTestSubmitter():
allHandlers.add('DaVinci', 'TestSubmitter', LHCbGaudiRunTimeHandler)
# Collect all of the prepared files which we want to pass along to this job
input_files, input_folders = collectPreparedFiles(app)
if input_folders:
raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future')
else:
for f in input_files:
input_sand.append(File(f))
logger.debug("input_sand: %s" % input_sand)
# It's this authors opinion that the script should be in the PATH on the WN
# As it stands policy is that is isn't so we have to call it in a relative way, hence "./"
c = StandardJobConfig('./'+scriptToRun.name, input_sand, [], output_sand)
return c
allHandlers.add('GaudiRun', 'Local', GaudiRunRTHandler)
#allHandlers.add('GaudiRun', 'Condor', GaudiRunRTHandler)
allHandlers.add('GaudiRun', 'Interactive', GaudiRunRTHandler)
allHandlers.add('GaudiRun', 'Batch', GaudiRunRTHandler)
allHandlers.add('GaudiRun', 'LSF', GaudiRunRTHandler)
def generateDiracInput(app):
"""
Construct a DIRAC input which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed
"""
input_files, input_folders = collectPreparedFiles(app)
job_command = prepareCommand(app)
# Generate the script which is to be executed for us on the WN
scriptToRun = generateWNScript(job_command, app)
input_sand.append(scriptToRun)
logger.debug("input_sand: %s" % input_sand)
# It's this authors opinion that the script should be in the PATH on the WN
# As it stands policy is that is isn't so we have to call it in a relative way, hence "./"
c = StandardJobConfig('./'+os.path.join(scriptToRun.subdir, scriptToRun.name), input_sand, [], output_sand)
return c
allHandlers.add('GaudiExec', 'Local', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'Condor', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'Interactive', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'Batch', GaudiExecRTHandler)
allHandlers.add('GaudiExec', 'LSF', GaudiExecRTHandler)
def generateDiracInput(app):
"""
Construct a DIRAC input which does not need to be unique to each job but is required to have a unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
"""
input_files, input_folders = collectPreparedFiles(app)
@staticmethod
def updateMonitoringInformation(jobs):
pass
class JobConfigObject:
def __init__(self,value):
self.value = value
def __str__(self):
return str(self.value)
def __repr__(self):
return self.__str__()
def getSandboxFiles(self):
return []
# This runtime handler just passes on the application configuration objects (adding a necessary wrapper around)
class NeutralRuntimeHandler(IRuntimeHandler):
def master_prepare(self, app, appmasterconfig):
r = appmasterconfig-5
return JobConfigObject(r)
def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig):
r = appsubconfig+jobmasterconfig.value-appmasterconfig
return JobConfigObject(r)
allHandlers.add('StoreTestApplication','StoreTestBackend',NeutralRuntimeHandler)
class RootLCGRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
runScript, inputsandbox, rootenv = downloadWrapper(app)
# propage command line args
argList = [str(s) for s in app.args]
return LCGJobConfig(runScript, inputsandbox, argList, app._getParent().outputsandbox, rootenv)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add("Root", "Cronus", RootRTHandler)
allHandlers.add("Root", "LSF", RootLocalRTHandler)
allHandlers.add("Root", "Local", RootLocalRTHandler)
allHandlers.add("Root", "Interactive", RootRTHandler)
# Easy way to test the system with ROOT dowloaded on demand
# allHandlers.add('Root','Local', RootDownloadHandler)
allHandlers.add("Root", "PBS", RootLocalRTHandler)
allHandlers.add("Root", "SGE", RootLocalRTHandler)
allHandlers.add("Root", "Condor", RootRTHandler)
allHandlers.add("Root", "LCG", RootLCGRTHandler)
allHandlers.add("Root", "CREAM", RootLCGRTHandler)
allHandlers.add("Root", "ARC", RootLCGRTHandler)
allHandlers.add("Root", "TestSubmitter", RootRTHandler)
allHandlers.add("Root", "Remote", RootLCGRTHandler)
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
if len(self.script) == 0:
logger.warning("No script defined. Will use a default "
'script which is probably not what you want.')
self.script = [File(os.path.join(
os.path.dirname(inspect.getsourcefile(GaudiPython)),
'options/GaudiPythonExample.py'))]
else:
for f in self.script:
f.name = fullpath(f.name)
return
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
# Associate the correct run-time handlers to GaudiPython for various backends.
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler
from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler
for backend in ['LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote']:
allHandlers.add('GaudiPython', backend, LHCbGaudiRunTimeHandler)
allHandlers.add('GaudiPython', 'Dirac', LHCbGaudiDiracRunTimeHandler)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
rt_handler = AthenaLocalRTHandler()
return rt_handler.prepare(app, appsubconfig, appmasterconfig,
jobmasterconfig)
def master_prepare(self, app, appmasterconfig):
be_name = app._getParent().backend.remote_backend._name
if be_name == "LCG" or be_name == "Condor" or be_name == "Cronus":
rt_handler = AthenaLCGRTHandler()
return rt_handler.master_prepare(app, appmasterconfig)
else:
rt_handler = AthenaLocalRTHandler()
return rt_handler.master_prepare(app, appmasterconfig)
allHandlers.add('Athena', 'Local', AthenaLocalRTHandler)
allHandlers.add('Athena', 'LSF', AthenaLocalRTHandler)
allHandlers.add('Athena', 'Condor', AthenaLocalRTHandler)
allHandlers.add('Athena', 'PBS', AthenaLocalRTHandler)
allHandlers.add('Athena', 'SGE', AthenaLocalRTHandler)
allHandlers.add('Athena', 'Remote', AthenaRemoteRTHandler)
config = getConfig('Athena')
configDQ2 = getConfig('DQ2')
configLCG = getConfig('LCG')
configSystem = getConfig('System')
logger = getLogger()
#$Log: not supported by cvs2svn $
#Revision 1.28 2009/07/17 07:32:12 elmsheus
#Fix dataset naming problem
############################################################################### # Ganga Project. http://cern.ch/ganga # # $Id: __init__.py,v 1.1 2008-10-04 17:42:38 karl Exp $ ############################################################################### # File: Classify/__init__.py # Author: K. Harrison # Created: 070122 """Package initialisation file""" __author__ = "K.Harrison <*****@*****.**>" __date__ = "22 February 2007" __version__ = "1.0" from Classify import Classify from ClassifyLCG import ClassifyLCG from ClassifyLocal import ClassifyLocal from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add( "Classify", "Interactive", ClassifyLocal ) allHandlers.add( "Classify", "Local", ClassifyLocal ) allHandlers.add( "Classify", "LCG", ClassifyLCG )
jid = job._getRoot().id
else:
jid = "%d" % job.id
environment["OUTPUT_JOBID"]=str(jid) # used for versionning
if app.dryrun:
environment["DRYRUN"] = "TRUE"
if app.dbrelease:
environment["ATLASDBREL"]=app.dbrelease
inputdata = []
filename="wrapper.sh"
exe = os.path.join(os.path.dirname(__file__),filename)
# output sandbox
outputbox =jobmasterconfig.outputbox
outputbox.extend(subjob_outbox)
return StandardJobConfig(File(exe),inputbox,args,outputbox,environment)
allHandlers.add('AthenaMC','Local',AthenaMCLocalRTHandler)
allHandlers.add('AthenaMC','LSF',AthenaMCLocalRTHandler)
#allHandlers.add('AthenaMC','Condor',AthenaMCLocalRTHandler) # condor has requirements...
allHandlers.add('AthenaMC','PBS',AthenaMCLocalRTHandler)
allHandlers.add('AthenaMC','SGE',AthenaMCLocalRTHandler)
config = getConfig('AthenaMC')
logger = getLogger('AthenaMC')
if isinstance(exe_cmd, list):
if path.isfile(path.abspath(exe_cmd[0])):
exe_cmd[0] = path.abspath(exe_cmd[0])
err = None
try:
rc = subprocess.call(exe_cmd, env=my_env, shell=True)
except Exception as x:
rc = -9999
print('Exception occured in running process: ' + exe_cmd)
print('Err was: ' + str(err))
subprocess.call('''['echo', '$PATH']''')
print('PATH: ' + str(my_env['PATH']))
print('PWD: ' + str(my_env['PWD']))
print("files on WN: " + str(listdir('.')))
raise
print("files on WN: " + str(listdir('.')))
###OUTPUTFILESINJECTEDCODE###
sys.exit(rc)
"""
return script_template
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Executable', 'Dirac', ExeDiracRTHandler)
python_wrapper = script_generator(python_wrapper,
remove_unreplaced=False,
FIXARGS='',
COMMAND='/usr/bin/env python %s %s' % (os.path.basename(app.script.name), '%s'),
JOINER=' ',
#INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'')
)
else:
python_wrapper = script_generator(python_wrapper,
remove_unreplaced=False,
FIXARGS='script_args=[formatVar(v) for v in script_args]',
COMMAND='export DISPLAY=\"localhoast:0.0\" && root -l -q \"%s(%s)\"' % (os.path.basename(app.script.name), '%s'),
JOINER=',',
#INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'')
)
f.write(python_wrapper)
f.close()
dirac_script = script_generator(lhcbdiracAPI_script_template(), **params)
return StandardJobConfig(dirac_script,
inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Root', 'Dirac', LHCbRootDiracRunTimeHandler)
def queueToDDM(queue):
from pandatools import Client
return Client.PandaSites[queue]['ddm']
class LCGRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
if not app.queue:
raise ApplicationConfigurationError(
None, 'queue not specified for PandaPilot')
args = app.arg_pattern.replace('%%QUEUE%%', app.queue)
job = app._getParent()
if not job.backend.requirements.sites:
job.backend.requirements.sites = [queueToDDM(app.queue)]
if not job.backend.requirements.sites:
raise ApplicationConfigurationError(
None, 'Could not map queue name to LCG site')
logger.info('Sending pilot for %s to %s' %
(app.queue, job.backend.requirements.sites))
return LCGJobConfig(app.exe,
app._getParent().inputsandbox, [args],
app._getParent().outputsandbox, None)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('PandaPilot', 'LCG', LCGRTHandler)
param += '-o "%s" ' % (
outfiles
) # must be double quotes, because python prints strings in 'single quotes'
for file in jspec.Files:
if file.type in ['output', 'log'] and configPanda['chirpconfig']:
file.dispatchDBlockToken = configPanda['chirpconfig']
logger.debug('chirp file %s', file)
jspec.jobParameters = param
return jspec
from Ganga.GPIDev.Credentials import GridProxy
gridProxy = GridProxy()
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Executable', 'Panda', ExecutablePandaRTHandler)
from Ganga.Utility.Config import getConfig, ConfigError
config = getConfig('Athena')
configDQ2 = getConfig('DQ2')
configPanda = getConfig('Panda')
from Ganga.Utility.logging import getLogger
logger = getLogger()
command : command to be used for file viewing
Return value: None
"""
#job = self.getJobObject()
#topdir = self.workdir.rstrip( os.sep )
#path = os.path.join( topdir, filename ).rstrip( os.sep )
#job.viewFile( path = path, command = command )
return None
# def remove_workdir(self):
# if config['remove_workdir']:
# import shutil
# try:
# shutil.rmtree(self.workdir)
# except OSError as x:
# logger.warning('problem removing the workdir %s: %s',str(self.id),str(x))
##
##############################################################################
##############################################################################
##
## Register a default runtime handler
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.Lib.Executable import RTHandler
allHandlers.add('Executable','SAGA', RTHandler)
# from Ganga.GPIDev.Base.Filters import allComponentFilters
# v = allComponentFilters['files'](v,self._schema.getItem('file_or_files'))
# if not type(v) is type([]):
# v = [v]
# return v
# # looks like in GPI we see everything double ;-)
# def _attribute_filter__get__(self,n,v):
# return v*2
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.Lib.Executable.Executable import RTHandler
class TestRTHandler(RTHandler):
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
if appconfig.find('error') != -1:
x = 'triggered failure during config'
raise Exception(x)
return RTHandler.prepare(self,app,appconfig,appmasterconfig,jobmasterconfig)
allHandlers.add('TestApplication','TestSubmitter',TestRTHandler)
## FIXME: Test for advanced properties...
## j = Job()
## a = TestAdvancedProperties()
## j.application = a
_schema = Executable._schema.inherit_copy()
_schema.datadict['called'] = SimpleItem(defvalue=False)
def __init__(self):
super(MockExeApplication, self).__init__()
self.called = False
def transition_update(self, new_status):
if new_status in expected_minimal_states:
expected_minimal_states.remove(new_status)
self.called = True
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('MockExeApplication', 'Local', RTHandler)
from Ganga.Utility.Plugin import allPlugins
allPlugins.add(MockExeApplication, 'applications', 'MockExeApplication')
from GangaTest.Framework.tests import GangaGPITestCase
from GangaTest.Framework.utils import sleep_until_completed
class TestTransitions(GangaGPITestCase):
def testTransitionsCalled(self):
m = MockExeApplication()
j = Job(backend=Local())
#if type(self.outputFiles[i]) is not list:
# logger.error('need a list of files! %s' % type(self.outputFiles[i]))
j.application.outputFiles=self.outputFiles[i]
if fin is True:
#if type(self.inputFiles[i]) is not list:
# logger.error('need a list of files! %s' % type(self.inputFiles[i]))
j.application.inputFiles=self.inputFiles[i]
if nMacro!=0:
j.application.ratMacro=self.ratMacro[i]
else:
j.application.prodScript=self.prodScript[i]
subjobs.append(j)
return subjobs
###################################################################
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaSNOplus.Lib.RTHandlers.RTRATProd import RTHandler, WGRTHandler, LCGRTHandler
allHandlers.add('RATProd','Local', RTHandler)
allHandlers.add('RATProd','PBS', RTHandler)
allHandlers.add('RATProd','SGE', RTHandler)
allHandlers.add('RATProd','Condor', RTHandler)
allHandlers.add('RATProd','LCG', LCGRTHandler)
allHandlers.add('RATProd','TestSubmitter', RTHandler)
allHandlers.add('RATProd','Interactive', RTHandler)
allHandlers.add('RATProd','Batch', RTHandler)
allHandlers.add('RATProd','WestGrid', WGRTHandler)
logger = Ganga.Utility.logging.getLogger()
jid = ""
if job._getRoot().subjobs:
jid = job._getRoot().id
else:
jid = "%d" % job.id
environment["OUTPUT_JOBID"] = str(jid) # used for versionning
if app.dryrun:
environment["DRYRUN"] = "TRUE"
if app.dbrelease:
environment["ATLASDBREL"] = app.dbrelease
inputdata = []
filename = "wrapper.sh"
exe = os.path.join(os.path.dirname(__file__), filename)
# output sandbox
outputbox = jobmasterconfig.outputbox
outputbox.extend(subjob_outbox)
return StandardJobConfig(File(exe), inputbox, args, outputbox,
environment)
allHandlers.add('AthenaMC', 'Local', AthenaMCLocalRTHandler)
allHandlers.add('AthenaMC', 'LSF', AthenaMCLocalRTHandler)
#allHandlers.add('AthenaMC','Condor',AthenaMCLocalRTHandler) # condor has requirements...
allHandlers.add('AthenaMC', 'PBS', AthenaMCLocalRTHandler)
allHandlers.add('AthenaMC', 'SGE', AthenaMCLocalRTHandler)
config = getConfig('AthenaMC')
logger = getLogger('AthenaMC')
share_path = os.path.join(share_dir, 'inputdata')
if not os.path.isdir(share_path):
os.makedirs(share_path)
f = open(os.path.join(share_path, 'options_data.pkl'), 'w+b')
pickle.dump(inputdata, f)
f.close()
# store the outputsandbox/outputdata defined in the options file
# Can remove this when no-longer need to define outputdata in optsfiles
# Can remove the if job: when look into how to do prepare for standalone app
# move into RuntimeHandler move whole parsing into options maybe?
# try and get the job object
# not present if preparing standalone app
# must change this as prepare should be seperate from the jpb.inputdata
share_path = os.path.join(share_dir, 'output')
if not os.path.isdir(share_path):
os.makedirs(share_path)
f = open(os.path.join(share_path, 'options_parser.pkl'), 'w+b')
pickle.dump(parser, f)
f.close()
#/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
for (backend, handler) in backend_handlers().iteritems():
allHandlers.add('AppName', backend, handler)
for i, rm in enumerate(self.ratMacro):
j = self.createSubjob(job)
j.application.ratMacro = rm
if self.outputFile != []:
j.application.outputFile = self.outputFile[i]
if self.inputFile != []:
j.application.inputFile = self.inputFile[i]
if self.nEvents != []:
j.application.nEvents = self.nEvents[i]
subjobs.append(j)
return subjobs
###################################################################
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaSNOplus.Lib.RTHandlers.RTRATUser import UserRTHandler, UserLCGRTHandler, UserWGRTHandler, UserDiracRTHandler
allHandlers.add("RATUser", "Local", UserRTHandler)
allHandlers.add("RATUser", "PBS", UserRTHandler)
allHandlers.add("RATUser", "SGE", UserRTHandler)
allHandlers.add("RATUser", "Condor", UserRTHandler)
allHandlers.add("RATUser", "LCG", UserLCGRTHandler)
allHandlers.add("RATUser", "Dirac", UserDiracRTHandler)
allHandlers.add("RATUser", "TestSubmitter", UserRTHandler)
allHandlers.add("RATUser", "Interactive", UserRTHandler)
allHandlers.add("RATUser", "Batch", UserRTHandler)
allHandlers.add("RATUser", "WestGrid", UserWGRTHandler)
logger = Ganga.Utility.logging.getLogger()
def _thread_main(self):
""" This is an internal function; the main loop of the background thread """
# Add runtime handlers for all the taskified applications, since now
# all the backends are loaded
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from .TaskApplication import handler_map
for basename, name in handler_map:
for backend in allHandlers.getAllBackends(basename):
allHandlers.add(name, backend,
allHandlers.get(basename, backend))
from Ganga.Core.GangaRepository import getRegistry
while getRegistry("jobs").hasStarted() is not True:
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
while True:
from Ganga.Core import monitoring_component
if (not monitoring_component is None
and monitoring_component.enabled
) or config['ForceTaskMonitoring']:
break
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
# setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
# and must go away soon
for tid in self.ids():
try:
self[tid]._getWriteAccess()
self[tid].startup()
except RegistryError:
continue
except Exception as err:
logger.error(
"Unknown/Unexpected Error in starting up tasks main loop")
logger.error("Exiting: err=%s" % str(err))
return
logger.debug("Entering main loop")
# Main loop
while self._main_thread is not None and not self._main_thread.should_stop(
):
# For each task try to run it
if config['ForceTaskMonitoring'] or monitoring_component.enabled:
for tid in self.ids():
logger.debug("Running over tid: %s" % str(tid))
try:
from Ganga.GPIDev.Lib.Tasks import ITask
if isType(self[tid], ITask):
# for new ITasks, always need write access
self[tid]._getWriteAccess()
p = self[tid]
else:
if self[tid].status in [
"running", "running/pause"
]:
self[tid]._getWriteAccess()
p = self[tid]
elif self[tid].status is 'completed' and (
self[tid].n_status('ready')
or self[tid].n_status('running')):
self[tid].updateStatus()
continue
else:
continue
except RegistryError:
# could not acquire lock
continue
if self._main_thread.should_stop():
break
try:
from Ganga.GPIDev.Lib.Tasks import ITask
if isType(self[tid], ITask):
# for new ITasks, always call update()
p.update()
else:
# TODO: Make this user-configurable and add better
# error message
if (p.n_status("failed") * 100.0 /
(20 + p.n_status("completed")) > 20):
p.pause()
logger.error(
"Task %s paused - %i jobs have failed while only %i jobs have completed successfully."
% (p.name, p.n_status("failed"),
p.n_status("completed")))
logger.error(
"Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()"
)
logger.error(
"You can then continue to run this task with tasks(%i).run()"
% p.id)
continue
numjobs = p.submitJobs()
if numjobs > 0:
self._flush([p])
# finalise any required transforms
p.finaliseTransforms()
p.updateStatus()
except Exception as x:
logger.error(
"Exception occurred in task monitoring loop: %s %s\nThe offending task was paused."
% (x.__class__, x))
type_, value_, traceback_ = sys.exc_info()
logger.error("Full traceback:\n %s" % ' '.join(
traceback.format_exception(type_, value_,
traceback_)))
p.pause()
if self._main_thread.should_stop():
break
if self._main_thread.should_stop():
break
logger.debug("TaskRegistry Sleeping for: %s seconds" %
str(config['TaskLoopFrequency']))
# Sleep interruptible for 10 seconds
for i in range(0, int(config['TaskLoopFrequency'] * 100)):
if self._main_thread.should_stop():
break
time.sleep(0.01)
############################################################################### # Ganga Project. http://cern.ch/ganga # # $Id: __init__.py,v 1.1 2008-10-04 17:42:39 karl Exp $ ############################################################################### # File: Vanseg/__init__.py # Author: K. Harrison # Created: 070122 """Package initialisation file""" __author__ = "K.Harrison <*****@*****.**>" __date__ = "22 February 2007" __version__ = "1.0" from Vanseg import Vanseg from VansegLCG import VansegLCG from VansegLocal import VansegLocal from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add( "Vanseg", "Interactive", VansegLocal ) allHandlers.add( "Vanseg", "Local", VansegLocal ) allHandlers.add( "Vanseg", "LCG", VansegLCG )
def submit(self,jobconfig,master_input_sandbox):
#if self.error.method == 'submit':
self.error.trigger(self._getParent().id)
return True
def kill(self):
#if self.error.method == 'kill':
self.error.trigger()
return True
def updateMonitoringInformation(jobs):
for j in jobs:
j.backend.error.trigger()
updateMonitoringInformation = staticmethod(updateMonitoringInformation)
class CrashRuntimeHandler(IRuntimeHandler):
def master_prepare(self, app, appmasterconfig):
#if appmasterconfig.method == 'master_prepare':
appmasterconfig.trigger()
return None
def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig):
#if appsubconfig.method == 'prepare':
appsubconfig.trigger()
return None
allHandlers.add('CrashTestApplication','CrashTestBackend',CrashRuntimeHandler)
return super(###CLASS###,self).get_dataset_name()
for method in ['getenv','getpack','make','cmt','register','add_output_dir','get_output_dir','add_dataset_name','get_dataset_name']:
setattr(eval(method), \"__doc__\", getattr(Gaudi, method).__doc__)
"""
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaBoss.Lib.Gaudi.GaudiRunTimeHandler import GaudiRunTimeHandler
from GangaBoss.Lib.DIRAC.GaudiDiracRTHandler import GaudiDiracRTHandler
for app in available_apps():
exec_str = class_str.replace('###CLASS###', app)
if app is 'Vetra':
lhcbra = os.path.expandvars("$Vetra_release_area")
exec_str = exec_str.replace('###SETLHCBRA###',
'self.lhcb_release_area = "%s"' % lhcbra)
else:
exec_str = exec_str.replace('###SETLHCBRA###', '')
if app is not 'Gaudi':
exec(exec_str)
for backend in ['LSF','Interactive','PBS','SGE','Local','Condor','Remote']:
allHandlers.add(app, backend, GaudiRunTimeHandler)
allHandlers.add(app, 'Dirac', GaudiDiracRTHandler)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
return LCGJobConfig(app._scriptname,
app._getParent().inputsandbox, [],
app._getParent().outputsandbox, app.env)
class gLiteRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
return gLiteJobConfig(app._scriptname,
app._getParent().inputsandbox, [],
app._getParent().outputsandbox, app.env)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Highland', 'LSF', RTHandler)
allHandlers.add('Highland', 'Local', RTHandler)
allHandlers.add('Highland', 'PBS', RTHandler)
allHandlers.add('Highland', 'SGE', RTHandler)
allHandlers.add('Highland', 'Condor', RTHandler)
allHandlers.add('Highland', 'LCG', LCGRTHandler)
allHandlers.add('Highland', 'gLite', gLiteRTHandler)
allHandlers.add('Highland', 'TestSubmitter', RTHandler)
allHandlers.add('Highland', 'Interactive', RTHandler)
allHandlers.add('Highland', 'Batch', RTHandler)
allHandlers.add('Highland', 'Cronus', RTHandler)
allHandlers.add('Highland', 'Remote', LCGRTHandler)
allHandlers.add('Highland', 'CREAM', LCGRTHandler)
# logger.error('need a list of files! %s' % type(self.outputFiles[i]))
j.application.outputFiles = self.outputFiles[i]
if fin is True:
#if type(self.inputFiles[i]) is not list:
# logger.error('need a list of files! %s' % type(self.inputFiles[i]))
j.application.inputFiles = self.inputFiles[i]
if nMacro != 0:
j.application.ratMacro = self.ratMacro[i]
else:
j.application.prodScript = self.prodScript[i]
subjobs.append(j)
return subjobs
###################################################################
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaSNOplus.Lib.RTHandlers.RTRATProd import RTHandler, WGRTHandler, LCGRTHandler
allHandlers.add('RATProd', 'Local', RTHandler)
allHandlers.add('RATProd', 'PBS', RTHandler)
allHandlers.add('RATProd', 'SGE', RTHandler)
allHandlers.add('RATProd', 'Condor', RTHandler)
allHandlers.add('RATProd', 'LCG', LCGRTHandler)
allHandlers.add('RATProd', 'TestSubmitter', RTHandler)
allHandlers.add('RATProd', 'Interactive', RTHandler)
allHandlers.add('RATProd', 'Batch', RTHandler)
allHandlers.add('RATProd', 'WestGrid', WGRTHandler)
logger = Ganga.Utility.logging.getLogger()
# Always check for None OR empty
#logger.info("self.module: %s" % str(self.module))
if isType(self.module, str):
self.module = File(self.module)
if self.module.name == None:
raise ApplicationConfigurationError(None, "Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(None, "Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(None, msg)
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
# Associate the correct run-time handlers to GaudiPython for various backends.
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler
from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler
for backend in ['LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote']:
allHandlers.add('Bender', backend, LHCbGaudiRunTimeHandler)
allHandlers.add('Bender', 'Dirac', LHCbGaudiDiracRunTimeHandler)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
############################################################################### # Ganga Project. http://cern.ch/ganga # # $Id: __init__.py,v 1.1 2009-05-10 16:42:03 karl Exp $ ############################################################################### # File: PDF/__init__.py # Author: K. Harrison # Created: 070122 """Package initialisation file""" __author__ = "K.Harrison <*****@*****.**>" __date__ = "9 May 2009" __version__ = "1.0" from PDF import PDF from PDF_LCG import PDF_LCG from PDF_Local import PDF_Local from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add( "PDF", "Interactive", PDF_Local ) allHandlers.add( "PDF", "Local", PDF_Local ) allHandlers.add( "PDF", "LCG", PDF_LCG )
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLSSL)
if matchURL != None:
param += " --sourceURL %s " % matchURL.group(1)
param += " --trf"
jspec.jobParameters = param
jspec.metadata="--trf \"%s\"" % arglist
#print "SUBJOB DETAILS:",jspec.values()
if app.dryrun:
print "job.application.dryrun activated, printing out job parameters"
print jspec.values()
return
return jspec
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('AthenaMC','Panda',AthenaMCPandaRTHandler)
from Ganga.GPIDev.Credentials import GridProxy
gridProxy = GridProxy()
from Ganga.Utility.Config import getConfig, ConfigError
config = getConfig('AthenaMC')
configDQ2 = getConfig('DQ2')
from Ganga.Utility.logging import getLogger
logger = getLogger()
]
## retrieve the FileStager log
if configDQ2['USE_ACCESS_INFO'] or (
job.inputdata and
(job.inputdata._name
in ['DQ2Dataset', 'AMIDataset', 'EventPicking'])
and job.inputdata.type in ['FILE_STAGER']):
outputbox += ['FileStager.out', 'FileStager.err']
if job.outputsandbox: outputbox += job.outputsandbox
# Switch for DEBUG print-out in logfiles
if app.useNoDebugLogs:
environment['GANGA_LOG_DEBUG'] = '0'
else:
environment['GANGA_LOG_DEBUG'] = '1'
return LCGJobConfig(File(exe), inputbox, [], outputbox, environment,
[], requirements)
allHandlers.add('Athena', 'LCG', AthenaLCGRTHandler)
allHandlers.add('Athena', 'CREAM', AthenaLCGRTHandler)
#allHandlers.add('Athena','Condor',AthenaLCGRTHandler)
allHandlers.add('Athena', 'Cronus', AthenaLCGRTHandler)
config = getConfig('Athena')
configDQ2 = getConfig('DQ2')
configSystem = getConfig('System')
logger = getLogger()
ifspec.GUID = guid
ifspec.fsize = size
ifspec.md5sum = checksum
ifspec.scope = scope
ifspec.dataset = jspec.prodDBlock
ifspec.prodDBlock = jspec.prodDBlock
ifspec.type = 'input'
jspec.addFile(ifspec)
if app.input_type:
itype = app.input_type
else:
itype = m.group(5)
if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"):
jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(job.inputdata.names))
else:
jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(job.inputdata.names))
# Log files.
lfspec = FileSpec()
lfspec.lfn = '%s.job.log.tgz' % jspec.jobName
lfspec.destinationDBlock = jspec.destinationDBlock
lfspec.destinationSE = jspec.destinationSE
lfspec.dataset = jspec.destinationDBlock
lfspec.type = 'log'
jspec.addFile(lfspec)
return jspec
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('ProdTrans', 'Panda', ProdTransPandaRTHandler)
if isinstance(exe_cmd, list):
if path.isfile(path.abspath(exe_cmd[0])):
exe_cmd[0] = path.abspath(exe_cmd[0])
err = None
try:
rc = subprocess.call(exe_cmd, env=my_env, shell=False)
except Exception as x:
rc = -9999
print('Exception occured in running process: ' + repr(exe_cmd))
print('Err was: ' + str(x))
subprocess.call('echo $PATH', shell=True)
print('PATH: ' + str(my_env['PATH']))
print('PWD: ' + str(my_env['PWD']))
print("files on WN: " + str(listdir('.')))
raise
print("files on WN: " + str(listdir('.')))
###OUTPUTFILESINJECTEDCODE###
sys.exit(rc)
"""
return script_template
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Executable', 'Dirac', ExeDiracRTHandler)
fout = FileSpec()
fout.lfn = outfiles[f]
fout.type = 'output'
fout.dataset = job.outputdata.datasetname
fout.destinationDBlock = job.outputdata.datasetname
fout.destinationSE = job.backend.site
jspec.addFile(fout)
param += '-o "%s" ' % (outfiles) # must be double quotes, because python prints strings in 'single quotes'
for file in jspec.Files:
if file.type in [ 'output', 'log'] and configPanda['chirpconfig']:
file.dispatchDBlockToken = configPanda['chirpconfig']
logger.debug('chirp file %s',file)
jspec.jobParameters = param
return jspec
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('Executable','Panda',ExecutablePandaRTHandler)
from Ganga.Utility.Config import getConfig, ConfigError
config = getConfig('Athena')
configDQ2 = getConfig('DQ2')
configPanda = getConfig('Panda')
from Ganga.Utility.logging import getLogger
logger = getLogger()
return (None,None)
config = getConfig('defaults_PandaPilot') #_Properties
config.options['exe'].type = type(None)
def queueToCE(queue):
from pandatools import Client
return Client.PandaSites[queue]['queue']+'-'+Client.PandaSites[queue]['localqueue']
def queueToDDM(queue):
from pandatools import Client
return Client.PandaSites[queue]['ddm']
class LCGRTHandler(IRuntimeHandler):
def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
if not app.queue:
raise ApplicationConfigurationError('queue not specified for PandaPilot')
args = app.arg_pattern.replace('%%QUEUE%%',app.queue)
job = app._getParent()
if not job.backend.requirements.sites:
job.backend.requirements.sites=[queueToDDM(app.queue)]
if not job.backend.requirements.sites:
raise ApplicationConfigurationError('Could not map queue name to LCG site')
logger.info('Sending pilot for %s to %s'%(app.queue,job.backend.requirements.sites))
return LCGJobConfig(app.exe,app._getParent().inputsandbox,[args],app._getParent().outputsandbox,None)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('PandaPilot','LCG', LCGRTHandler)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
return LCGJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env)
class gLiteRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
return gLiteJobConfig(
app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env
)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add("runND280SandMC", "LSF", RTHandler)
allHandlers.add("runND280SandMC", "Local", RTHandler)
allHandlers.add("runND280SandMC", "PBS", RTHandler)
allHandlers.add("runND280SandMC", "SGE", RTHandler)
allHandlers.add("runND280SandMC", "Condor", RTHandler)
allHandlers.add("runND280SandMC", "LCG", LCGRTHandler)
allHandlers.add("runND280SandMC", "gLite", gLiteRTHandler)
allHandlers.add("runND280SandMC", "TestSubmitter", RTHandler)
allHandlers.add("runND280SandMC", "Interactive", RTHandler)
allHandlers.add("runND280SandMC", "Batch", RTHandler)
allHandlers.add("runND280SandMC", "Cronus", RTHandler)
allHandlers.add("runND280SandMC", "Remote", LCGRTHandler)
allHandlers.add("runND280SandMC", "CREAM", LCGRTHandler)
'no_summary': SimpleItem(defvalue=[], sequence=1, summary_sequence_maxlen=-1, typelist=['str']),
'seq': SimpleItem(defvalue=[], sequence=1, typelist=['int']),
'gList': SimpleItem(defvalue=[], sequence=1, typelist=['str']),
'gListComp': ComponentItem('files', defvalue=[], sequence=1),
'simple_print': SimpleItem(defvalue='', summary_print='_print_summary_simple_print'),
'comp_print': ComponentItem('backends', defvalue=None, summary_print='_print_summary_comp_print'),
'is_prepared': SimpleItem(defvalue=None, strict_sequence=0, visitable=1, copyable=1, hidden=0, typelist=['type(None)', 'bool', ShareDir], protected=0, comparable=1, doc='Location of shared resources. Presence of this attribute implies the application has been prepared.'),
})
def configure(self, master_appconfig):
return (None, None)
def _print_summary_bound_comp(self, value, verbosity_level):
return '_print_summary_bound_comp'
def _print_summary_bound_simple(self, value, verbosity_level):
return '_print_summary_bound_simple'
def _print_summary_simple_print(self, value, verbosity_level):
return '_print_summary_simple_print'
def _print_summary_comp_print(self, value, verbosity_level):
return '_print_summary_comp_print'
class Handler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
return 0
allHandlers.add('GListApp', 'TestSubmitter', Handler)
raise ApplicationConfigurationError(
"Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
"Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(msg)
def postprocess(self):
XMLPostProcessor.postprocess(self, logger)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
# Associate the correct run-time handlers to GaudiPython for various backends.
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler
from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler
for backend in [
'LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote'
]:
allHandlers.add('Bender', backend, LHCbGaudiRunTimeHandler)
allHandlers.add('Bender', 'Dirac', LHCbGaudiDiracRunTimeHandler)
#\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
def _thread_main(self):
""" This is an internal function; the main loop of the background thread """
# Add runtime handlers for all the taskified applications, since now
# all the backends are loaded
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from .TaskApplication import handler_map
for basename, name in handler_map:
for backend in allHandlers.getAllBackends(basename):
allHandlers.add(
name, backend, allHandlers.get(basename, backend))
from Ganga.Core.GangaRepository import getRegistry
while getRegistry("jobs").hasStarted() is not True:
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
while True:
from Ganga.Core import monitoring_component
if (not monitoring_component is None and monitoring_component.enabled) or config['ForceTaskMonitoring']:
break
time.sleep(0.1)
if self._main_thread is None or self._main_thread.should_stop():
return
# setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY
# and must go away soon
for tid in self.ids():
try:
self[tid]._getWriteAccess()
self[tid].startup()
except RegistryError:
continue
except Exception as err:
logger.error("Unknown/Unexpected Error in starting up tasks main loop")
logger.error("Exiting: err=%s" % str(err))
return
logger.debug("Entering main loop")
# Main loop
while self._main_thread is not None and not self._main_thread.should_stop():
# For each task try to run it
if config['ForceTaskMonitoring'] or monitoring_component.enabled:
for tid in self.ids():
logger.debug("Running over tid: %s" % str(tid))
try:
from Ganga.GPIDev.Lib.Tasks import ITask
if isType(self[tid], ITask):
# for new ITasks, always need write access
self[tid]._getWriteAccess()
p = self[tid]
else:
if self[tid].status in ["running", "running/pause"]:
self[tid]._getWriteAccess()
p = self[tid]
elif self[tid].status is 'completed' and (self[tid].n_status('ready') or self[tid].n_status('running')):
self[tid].updateStatus()
continue
else:
continue
except RegistryError:
# could not acquire lock
continue
if self._main_thread.should_stop():
break
try:
from Ganga.GPIDev.Lib.Tasks import ITask
if isType(self[tid], ITask):
# for new ITasks, always call update()
p.update()
else:
# TODO: Make this user-configurable and add better
# error message
if (p.n_status("failed") * 100.0 / (20 + p.n_status("completed")) > 20):
p.pause()
logger.error("Task %s paused - %i jobs have failed while only %i jobs have completed successfully." % (
p.name, p.n_status("failed"), p.n_status("completed")))
logger.error(
"Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()")
logger.error(
"You can then continue to run this task with tasks(%i).run()" % p.id)
continue
numjobs = p.submitJobs()
if numjobs > 0:
self._flush([p])
# finalise any required transforms
p.finaliseTransforms()
p.updateStatus()
except Exception as x:
logger.error(
"Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x))
type_, value_, traceback_ = sys.exc_info()
logger.error("Full traceback:\n %s" % ' '.join(
traceback.format_exception(type_, value_, traceback_)))
p.pause()
if self._main_thread.should_stop():
break
if self._main_thread.should_stop():
break
logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency']))
# Sleep interruptible for 10 seconds
for i in range(0, int(config['TaskLoopFrequency'] * 100)):
if self._main_thread.should_stop():
break
time.sleep(0.01)
return LCGJobConfig(app._scriptname,
app._getParent().inputsandbox, [],
app._getParent().outputsandbox, app.env)
class gLiteRTHandler(IRuntimeHandler):
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.gLite import gLiteJobConfig
return gLiteJobConfig(app._scriptname,
app._getParent().inputsandbox, [],
app._getParent().outputsandbox, app.env)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
allHandlers.add('oaReconPlusoaAnalysis', 'LSF', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'Local', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'PBS', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'SGE', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'Condor', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'LCG', LCGRTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'gLite', gLiteRTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'TestSubmitter', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'Interactive', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'Batch', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'Cronus', RTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'Remote', LCGRTHandler)
allHandlers.add('oaReconPlusoaAnalysis', 'CREAM', LCGRTHandler)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
from Ganga.GPIDev.Adapters.IRuntimeHandler import IRuntimeHandler
class CRABRuntimeHandler(IRuntimeHandler):
"""Almost empty runtime handler."""
def master_prepare(self, app, appconfig):
"""No need to prepare this jobs."""
return None
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig):
"""No need to prepare this jobs."""
return None
allHandlers.add('CRABApp', 'CRABBackend', CRABRuntimeHandler)
