ifspec.dataset = jspec.prodDBlock ifspec.prodDBlock = jspec.prodDBlock ifspec.type = 'input' jspec.addFile(ifspec) if app.input_type: itype = app.input_type else: itype = m.group(5) if jspec.transformation.endswith( "_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join( job.inputdata.names)) else: jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join( job.inputdata.names)) # Log files. lfspec = FileSpec() lfspec.lfn = '%s.job.log.tgz' % jspec.jobName lfspec.destinationDBlock = jspec.destinationDBlock lfspec.destinationSE = jspec.destinationSE lfspec.dataset = jspec.destinationDBlock lfspec.type = 'log' jspec.addFile(lfspec) return jspec from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('ProdTrans', 'Panda', ProdTransPandaRTHandler)
class RootLCGRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig runScript, inputsandbox, rootenv = downloadWrapper(app) # propage command line args argList = [str(s) for s in app.args] return LCGJobConfig(runScript, inputsandbox, argList, app._getParent().outputsandbox, rootenv) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Root', 'Cronus', RootRTHandler) allHandlers.add('Root', 'LSF', RootLocalRTHandler) allHandlers.add('Root', 'Local', RootLocalRTHandler) allHandlers.add('Root', 'Interactive', RootRTHandler) # Easy way to test the system with ROOT dowloaded on demand #allHandlers.add('Root','Local', RootDownloadHandler) allHandlers.add('Root', 'PBS', RootLocalRTHandler) allHandlers.add('Root', 'SGE', RootLocalRTHandler) allHandlers.add('Root', 'Condor', RootRTHandler) allHandlers.add('Root', 'LCG', RootLCGRTHandler) allHandlers.add('Root', 'CREAM', RootLCGRTHandler) allHandlers.add('Root', 'ARC', RootLCGRTHandler) allHandlers.add('Root', 'TestSubmitter', RootRTHandler) allHandlers.add('Root', 'Remote', RootLCGRTHandler)
Return value: None """ # job = self.getJobObject() # topdir = self.workdir.rstrip( os.sep ) # path = os.path.join( topdir, filename ).rstrip( os.sep ) # job.viewFile( path = path, command = command ) return None # def remove_workdir(self): # if config['remove_workdir']: # import shutil # try: # shutil.rmtree(self.workdir) # except OSError as x: # logger.warning('problem removing the workdir %s: %s',str(self.id),str(x)) ## ############################################################################## ############################################################################## ## ## Register a default runtime handler from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.Lib.Executable import RTHandler allHandlers.add("Executable", "SAGA", RTHandler)
COMMAND='/usr/bin/env python %s %s' % (os.path.basename(app.script.name), '%s'), JOINER=' ', #INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'') ) else: python_wrapper = script_generator( python_wrapper, remove_unreplaced=False, FIXARGS='script_args=[formatVar(v) for v in script_args]', COMMAND= 'export DISPLAY=\"localhoast:0.0\" && root -l -q \"%s(%s)\"' % (os.path.basename(app.script.name), '%s'), JOINER=',', #INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'') ) f.write(python_wrapper) f.close() dirac_script = script_generator(lhcbdiracAPI_script_template(), **params) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox)) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Root', 'Dirac', LHCbRootDiracRunTimeHandler)
return rc # Main if __name__ == '__main__': err = None try: rc = run_Im3ShapeApp() except Exception as x: rc = -9999 print('Exception occured in running app.') print('Err was: ' + str(x)) print("files on WN: " + str(listdir('.'))) print("environ: %s" % environ) raise print("files on WN: " + str(listdir('.'))) ###OUTPUTFILESINJECTEDCODE### exit(rc) """ return script_template # \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add("Im3ShapeApp", "Dirac", Im3ShapeDiracRTHandler)
app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) # else assume it's a system binary, so we don't need to # transport anything to the sharedir else: prepared_exe = app.exe elif isinstance(app.exe, File): logger.info("Submitting a prepared application; taking any input files from %s" % ( app.is_prepared.name)) prepared_exe = File(os.path.join(os.path.join( shared_path, app.is_prepared.name), os.path.basename(File(app.exe).name))) return gLiteJobConfig(prepared_exe, app._getParent().inputsandbox, convertIntToStringArgs(app.args), app._getParent().outputsandbox, app.env) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Executable', 'LSF', RTHandler) allHandlers.add('Executable', 'Local', RTHandler) allHandlers.add('Executable', 'PBS', RTHandler) allHandlers.add('Executable', 'SGE', RTHandler) allHandlers.add('Executable', 'Condor', RTHandler) allHandlers.add('Executable', 'LCG', LCGRTHandler) allHandlers.add('Executable', 'gLite', gLiteRTHandler) allHandlers.add('Executable', 'TestSubmitter', RTHandler) allHandlers.add('Executable', 'Interactive', RTHandler) allHandlers.add('Executable', 'Batch', RTHandler) allHandlers.add('Executable', 'Cronus', RTHandler) allHandlers.add('Executable', 'Remote', LCGRTHandler) allHandlers.add('Executable', 'CREAM', LCGRTHandler) allHandlers.add('Executable', 'ARC', LCGRTHandler)
filename : name of file to be viewed => Path specified relative to work directory command : command to be used for file viewing Return value: None """ #job = self.getJobObject() #topdir = self.workdir.rstrip( os.sep ) #path = os.path.join( topdir, filename ).rstrip( os.sep ) #job.viewFile( path = path, command = command ) return None # def remove_workdir(self): # if config['remove_workdir']: # import shutil # try: # shutil.rmtree(self.workdir) # except OSError as x: # logger.warning('problem removing the workdir %s: %s',str(self.id),str(x)) ## ############################################################################## ############################################################################## ## ## Register a default runtime handler from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.Lib.Executable import RTHandler allHandlers.add('Executable', 'SAGA', RTHandler)
# Generate the script which is to be executed for us on the WN scriptToRun = generateWNScript(job_command, app) input_sand.append(scriptToRun) logger.debug("input_sand: %s" % input_sand) # It's this authors opinion that the script should be in the PATH on the WN # As it stands policy is that is isn't so we have to call it in a relative way, hence "./" c = StandardJobConfig( './' + os.path.join(scriptToRun.subdir, scriptToRun.name), input_sand, [], output_sand) return c allHandlers.add('GaudiExec', 'Local', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'Condor', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'Interactive', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'Batch', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'LSF', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'PBS', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'SGE', GaudiExecRTHandler) def generateDiracInput(app): """ Construct a DIRAC input which does not need to be unique to each job but is required to have a unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed """
from pandatools import Client from pandatools import AthenaUtils from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec job = app._getParent() masterjob = job._getRoot() logger.debug('AthenaJediRTHandler prepare called for %s', job.getFQID('.')) # in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter return {} from Ganga.GPIDev.Credentials import GridProxy gridProxy = GridProxy() from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers for app in ['Athena', 'ExecutableDQ2', 'RootDQ2']: allHandlers.add(app, 'Jedi', AthenaJediRTHandler) from Ganga.Utility.Config import getConfig, ConfigError configDQ2 = getConfig('DQ2') configJedi = getConfig('Jedi') from Ganga.Utility.logging import getLogger logger = getLogger()
return c class LCGRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig c = LCGJobConfig(app.exe, app.inputs, app.args, app.outputs, app.envs) c.monitoring_svc = mc['GangaTutorial'] return c from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('PrimeFactorizer', 'LSF', RTHandler) allHandlers.add('PrimeFactorizer', 'Local', RTHandler) allHandlers.add('PrimeFactorizer', 'PBS', RTHandler) allHandlers.add('PrimeFactorizer', 'SGE', RTHandler) allHandlers.add('PrimeFactorizer', 'Condor', RTHandler) allHandlers.add('PrimeFactorizer', 'LCG', LCGRTHandler) allHandlers.add('PrimeFactorizer', 'TestSubmitter', RTHandler) allHandlers.add('PrimeFactorizer', 'Interactive', RTHandler) allHandlers.add('PrimeFactorizer', 'Batch', RTHandler) ############################################################## ## handler for NG from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.GPIDev.Adapters.IRuntimeHandler import IRuntimeHandler
# from Ganga.GPIDev.Base.Filters import allComponentFilters # v = allComponentFilters['files'](v,self._schema.getItem('file_or_files')) # if not type(v) is type([]): # v = [v] # return v # # looks like in GPI we see everything double ;-) # def _attribute_filter__get__(self,n,v): # return v*2 from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.Lib.Executable.Executable import RTHandler class TestRTHandler(RTHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): if appconfig.find('error') != -1: x = 'triggered failure during config' raise Exception(x) return RTHandler.prepare(self, app, appconfig, appmasterconfig, jobmasterconfig) allHandlers.add('TestApplication', 'TestSubmitter', TestRTHandler) ## FIXME: Test for advanced properties... ## j = Job() ## a = TestAdvancedProperties() ## j.application = a
File( os.path.join( os.path.dirname(inspect.getsourcefile(GaudiPython)), 'options/GaudiPythonExample.py')) ] else: for f in self.script: f.name = fullpath(f.name) return def postprocess(self): XMLPostProcessor.postprocess(self, logger) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# # Associate the correct run-time handlers to GaudiPython for various backends. from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler for backend in [ 'LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote' ]: allHandlers.add('GaudiPython', backend, LHCbGaudiRunTimeHandler) allHandlers.add('GaudiPython', 'Dirac', LHCbGaudiDiracRunTimeHandler) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
for i,rm in enumerate(self.ratMacro): j = self.createSubjob(job) j.application.ratMacro = rm if self.outputFile!=[]: j.application.outputFile = self.outputFile[i] if self.inputFile!=[]: j.application.inputFile = self.inputFile[i] if self.nEvents!=[]: j.application.nEvents = self.nEvents[i] subjobs.append(j) return subjobs ################################################################### from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaSNOplus.Lib.RTHandlers.RTRATUser import UserRTHandler, UserLCGRTHandler, UserWGRTHandler, UserDiracRTHandler allHandlers.add('RATUser','Local', UserRTHandler) allHandlers.add('RATUser','PBS', UserRTHandler) allHandlers.add('RATUser','SGE', UserRTHandler) allHandlers.add('RATUser','Condor', UserRTHandler) allHandlers.add('RATUser','LCG', UserLCGRTHandler) allHandlers.add('RATUser','Dirac', UserDiracRTHandler) allHandlers.add('RATUser','TestSubmitter', UserRTHandler) allHandlers.add('RATUser','Interactive', UserRTHandler) allHandlers.add('RATUser','Batch', UserRTHandler) allHandlers.add('RATUser','WestGrid', UserWGRTHandler) logger = Ganga.Utility.logging.getLogger()
uploadSources(os.path.dirname(self.inputsandbox),os.path.basename(self.inputsandbox)) if not self.inputsandbox == tmp_user_area_name: logger.info('Removing source tarball %s ...' % self.inputsandbox ) os.remove(self.inputsandbox) return taskParamMap def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig): '''prepare the subjob specific configuration''' job = app._getParent() logger.debug('AthenaJediRTHandler prepare called for %s', job.getFQID('.')) # in case of a simple job get the dataset content, otherwise subjobs are filled by the splitter return {} from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers for app in ['Athena', 'ExecutableDQ2', 'RootDQ2']: allHandlers.add(app,'Jedi',AthenaJediRTHandler) from Ganga.Utility.Config import getConfig, ConfigError configDQ2 = getConfig('DQ2') configJedi = getConfig('Jedi') from Ganga.Utility.logging import getLogger logger = getLogger()
path = tempfile.mkdtemp(prefix='GANGA_') logger.info( 'Bender application will be prepared in the temporary directory %s' % path) the_path = prepare_cmake_app(project, version, path) # scripts = kwarsg.pop('scripts', []) if script: scripts.append(script) # from Ganga.GPI import OstapRun as _OR ## black magic return _OR(scripts=scripts, directory=the_path, **kwargs) ## export it! from Ganga.Runtime.GPIexport import exportToGPI exportToGPI('prepareBender', prepareBender, 'Functions') exportToGPI('prepareBenderRun', prepareBenderRun, 'Functions') exportToGPI('prepareOstapRun', prepareOstapRun, 'Functions') ## for app in ('BenderModule', 'BenderRun', 'OstapRun'): allHandlers.add(app, 'Dirac', GaudiExecDiracRTHandler) for backend in ("Local", "Interactive", "LSF", "PBS", "SGE", "Condor"): allHandlers.add(app, backend, GaudiExecRTHandler) # ============================================================================= # The END # =============================================================================
def addLocalTestSubmitter(): allHandlers.add('DaVinci', 'TestSubmitter', LHCbGaudiRunTimeHandler)
# Collect all of the prepared files which we want to pass along to this job input_files, input_folders = collectPreparedFiles(app) if input_folders: raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future') else: for f in input_files: input_sand.append(File(f)) logger.debug("input_sand: %s" % input_sand) # It's this authors opinion that the script should be in the PATH on the WN # As it stands policy is that is isn't so we have to call it in a relative way, hence "./" c = StandardJobConfig('./'+scriptToRun.name, input_sand, [], output_sand) return c allHandlers.add('GaudiRun', 'Local', GaudiRunRTHandler) #allHandlers.add('GaudiRun', 'Condor', GaudiRunRTHandler) allHandlers.add('GaudiRun', 'Interactive', GaudiRunRTHandler) allHandlers.add('GaudiRun', 'Batch', GaudiRunRTHandler) allHandlers.add('GaudiRun', 'LSF', GaudiRunRTHandler) def generateDiracInput(app): """ Construct a DIRAC input which must be unique to each job to have unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed """ input_files, input_folders = collectPreparedFiles(app)
job_command = prepareCommand(app) # Generate the script which is to be executed for us on the WN scriptToRun = generateWNScript(job_command, app) input_sand.append(scriptToRun) logger.debug("input_sand: %s" % input_sand) # It's this authors opinion that the script should be in the PATH on the WN # As it stands policy is that is isn't so we have to call it in a relative way, hence "./" c = StandardJobConfig('./'+os.path.join(scriptToRun.subdir, scriptToRun.name), input_sand, [], output_sand) return c allHandlers.add('GaudiExec', 'Local', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'Condor', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'Interactive', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'Batch', GaudiExecRTHandler) allHandlers.add('GaudiExec', 'LSF', GaudiExecRTHandler) def generateDiracInput(app): """ Construct a DIRAC input which does not need to be unique to each job but is required to have a unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed """ input_files, input_folders = collectPreparedFiles(app)
@staticmethod def updateMonitoringInformation(jobs): pass class JobConfigObject: def __init__(self,value): self.value = value def __str__(self): return str(self.value) def __repr__(self): return self.__str__() def getSandboxFiles(self): return [] # This runtime handler just passes on the application configuration objects (adding a necessary wrapper around) class NeutralRuntimeHandler(IRuntimeHandler): def master_prepare(self, app, appmasterconfig): r = appmasterconfig-5 return JobConfigObject(r) def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig): r = appsubconfig+jobmasterconfig.value-appmasterconfig return JobConfigObject(r) allHandlers.add('StoreTestApplication','StoreTestBackend',NeutralRuntimeHandler)
class RootLCGRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig runScript, inputsandbox, rootenv = downloadWrapper(app) # propage command line args argList = [str(s) for s in app.args] return LCGJobConfig(runScript, inputsandbox, argList, app._getParent().outputsandbox, rootenv) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add("Root", "Cronus", RootRTHandler) allHandlers.add("Root", "LSF", RootLocalRTHandler) allHandlers.add("Root", "Local", RootLocalRTHandler) allHandlers.add("Root", "Interactive", RootRTHandler) # Easy way to test the system with ROOT dowloaded on demand # allHandlers.add('Root','Local', RootDownloadHandler) allHandlers.add("Root", "PBS", RootLocalRTHandler) allHandlers.add("Root", "SGE", RootLocalRTHandler) allHandlers.add("Root", "Condor", RootRTHandler) allHandlers.add("Root", "LCG", RootLCGRTHandler) allHandlers.add("Root", "CREAM", RootLCGRTHandler) allHandlers.add("Root", "ARC", RootLCGRTHandler) allHandlers.add("Root", "TestSubmitter", RootRTHandler) allHandlers.add("Root", "Remote", RootLCGRTHandler)
def _check_inputs(self): """Checks the validity of user's entries for GaudiPython schema""" if len(self.script) == 0: logger.warning("No script defined. Will use a default " 'script which is probably not what you want.') self.script = [File(os.path.join( os.path.dirname(inspect.getsourcefile(GaudiPython)), 'options/GaudiPythonExample.py'))] else: for f in self.script: f.name = fullpath(f.name) return def postprocess(self): XMLPostProcessor.postprocess(self, logger) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# # Associate the correct run-time handlers to GaudiPython for various backends. from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler for backend in ['LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote']: allHandlers.add('GaudiPython', backend, LHCbGaudiRunTimeHandler) allHandlers.add('GaudiPython', 'Dirac', LHCbGaudiDiracRunTimeHandler) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
rt_handler = AthenaLocalRTHandler() return rt_handler.prepare(app, appsubconfig, appmasterconfig, jobmasterconfig) def master_prepare(self, app, appmasterconfig): be_name = app._getParent().backend.remote_backend._name if be_name == "LCG" or be_name == "Condor" or be_name == "Cronus": rt_handler = AthenaLCGRTHandler() return rt_handler.master_prepare(app, appmasterconfig) else: rt_handler = AthenaLocalRTHandler() return rt_handler.master_prepare(app, appmasterconfig) allHandlers.add('Athena', 'Local', AthenaLocalRTHandler) allHandlers.add('Athena', 'LSF', AthenaLocalRTHandler) allHandlers.add('Athena', 'Condor', AthenaLocalRTHandler) allHandlers.add('Athena', 'PBS', AthenaLocalRTHandler) allHandlers.add('Athena', 'SGE', AthenaLocalRTHandler) allHandlers.add('Athena', 'Remote', AthenaRemoteRTHandler) config = getConfig('Athena') configDQ2 = getConfig('DQ2') configLCG = getConfig('LCG') configSystem = getConfig('System') logger = getLogger() #$Log: not supported by cvs2svn $ #Revision 1.28 2009/07/17 07:32:12 elmsheus #Fix dataset naming problem
############################################################################### # Ganga Project. http://cern.ch/ganga # # $Id: __init__.py,v 1.1 2008-10-04 17:42:38 karl Exp $ ############################################################################### # File: Classify/__init__.py # Author: K. Harrison # Created: 070122 """Package initialisation file""" __author__ = "K.Harrison <*****@*****.**>" __date__ = "22 February 2007" __version__ = "1.0" from Classify import Classify from ClassifyLCG import ClassifyLCG from ClassifyLocal import ClassifyLocal from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add( "Classify", "Interactive", ClassifyLocal ) allHandlers.add( "Classify", "Local", ClassifyLocal ) allHandlers.add( "Classify", "LCG", ClassifyLCG )
jid = job._getRoot().id else: jid = "%d" % job.id environment["OUTPUT_JOBID"]=str(jid) # used for versionning if app.dryrun: environment["DRYRUN"] = "TRUE" if app.dbrelease: environment["ATLASDBREL"]=app.dbrelease inputdata = [] filename="wrapper.sh" exe = os.path.join(os.path.dirname(__file__),filename) # output sandbox outputbox =jobmasterconfig.outputbox outputbox.extend(subjob_outbox) return StandardJobConfig(File(exe),inputbox,args,outputbox,environment) allHandlers.add('AthenaMC','Local',AthenaMCLocalRTHandler) allHandlers.add('AthenaMC','LSF',AthenaMCLocalRTHandler) #allHandlers.add('AthenaMC','Condor',AthenaMCLocalRTHandler) # condor has requirements... allHandlers.add('AthenaMC','PBS',AthenaMCLocalRTHandler) allHandlers.add('AthenaMC','SGE',AthenaMCLocalRTHandler) config = getConfig('AthenaMC') logger = getLogger('AthenaMC')
if isinstance(exe_cmd, list): if path.isfile(path.abspath(exe_cmd[0])): exe_cmd[0] = path.abspath(exe_cmd[0]) err = None try: rc = subprocess.call(exe_cmd, env=my_env, shell=True) except Exception as x: rc = -9999 print('Exception occured in running process: ' + exe_cmd) print('Err was: ' + str(err)) subprocess.call('''['echo', '$PATH']''') print('PATH: ' + str(my_env['PATH'])) print('PWD: ' + str(my_env['PWD'])) print("files on WN: " + str(listdir('.'))) raise print("files on WN: " + str(listdir('.'))) ###OUTPUTFILESINJECTEDCODE### sys.exit(rc) """ return script_template #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Executable', 'Dirac', ExeDiracRTHandler)
python_wrapper = script_generator(python_wrapper, remove_unreplaced=False, FIXARGS='', COMMAND='/usr/bin/env python %s %s' % (os.path.basename(app.script.name), '%s'), JOINER=' ', #INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'') ) else: python_wrapper = script_generator(python_wrapper, remove_unreplaced=False, FIXARGS='script_args=[formatVar(v) for v in script_args]', COMMAND='export DISPLAY=\"localhoast:0.0\" && root -l -q \"%s(%s)\"' % (os.path.basename(app.script.name), '%s'), JOINER=',', #INJECTEDCODE = getWNCodeForOutputPostprocessing(job,'') ) f.write(python_wrapper) f.close() dirac_script = script_generator(lhcbdiracAPI_script_template(), **params) return StandardJobConfig(dirac_script, inputbox=unique(inputsandbox), outputbox=unique(outputsandbox)) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Root', 'Dirac', LHCbRootDiracRunTimeHandler)
def queueToDDM(queue): from pandatools import Client return Client.PandaSites[queue]['ddm'] class LCGRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig if not app.queue: raise ApplicationConfigurationError( None, 'queue not specified for PandaPilot') args = app.arg_pattern.replace('%%QUEUE%%', app.queue) job = app._getParent() if not job.backend.requirements.sites: job.backend.requirements.sites = [queueToDDM(app.queue)] if not job.backend.requirements.sites: raise ApplicationConfigurationError( None, 'Could not map queue name to LCG site') logger.info('Sending pilot for %s to %s' % (app.queue, job.backend.requirements.sites)) return LCGJobConfig(app.exe, app._getParent().inputsandbox, [args], app._getParent().outputsandbox, None) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('PandaPilot', 'LCG', LCGRTHandler)
param += '-o "%s" ' % ( outfiles ) # must be double quotes, because python prints strings in 'single quotes' for file in jspec.Files: if file.type in ['output', 'log'] and configPanda['chirpconfig']: file.dispatchDBlockToken = configPanda['chirpconfig'] logger.debug('chirp file %s', file) jspec.jobParameters = param return jspec from Ganga.GPIDev.Credentials import GridProxy gridProxy = GridProxy() from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Executable', 'Panda', ExecutablePandaRTHandler) from Ganga.Utility.Config import getConfig, ConfigError config = getConfig('Athena') configDQ2 = getConfig('DQ2') configPanda = getConfig('Panda') from Ganga.Utility.logging import getLogger logger = getLogger()
command : command to be used for file viewing Return value: None """ #job = self.getJobObject() #topdir = self.workdir.rstrip( os.sep ) #path = os.path.join( topdir, filename ).rstrip( os.sep ) #job.viewFile( path = path, command = command ) return None # def remove_workdir(self): # if config['remove_workdir']: # import shutil # try: # shutil.rmtree(self.workdir) # except OSError as x: # logger.warning('problem removing the workdir %s: %s',str(self.id),str(x)) ## ############################################################################## ############################################################################## ## ## Register a default runtime handler from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.Lib.Executable import RTHandler allHandlers.add('Executable','SAGA', RTHandler)
# from Ganga.GPIDev.Base.Filters import allComponentFilters # v = allComponentFilters['files'](v,self._schema.getItem('file_or_files')) # if not type(v) is type([]): # v = [v] # return v # # looks like in GPI we see everything double ;-) # def _attribute_filter__get__(self,n,v): # return v*2 from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.Lib.Executable.Executable import RTHandler class TestRTHandler(RTHandler): def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig): if appconfig.find('error') != -1: x = 'triggered failure during config' raise Exception(x) return RTHandler.prepare(self,app,appconfig,appmasterconfig,jobmasterconfig) allHandlers.add('TestApplication','TestSubmitter',TestRTHandler) ## FIXME: Test for advanced properties... ## j = Job() ## a = TestAdvancedProperties() ## j.application = a
_schema = Executable._schema.inherit_copy() _schema.datadict['called'] = SimpleItem(defvalue=False) def __init__(self): super(MockExeApplication, self).__init__() self.called = False def transition_update(self, new_status): if new_status in expected_minimal_states: expected_minimal_states.remove(new_status) self.called = True from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('MockExeApplication', 'Local', RTHandler) from Ganga.Utility.Plugin import allPlugins allPlugins.add(MockExeApplication, 'applications', 'MockExeApplication') from GangaTest.Framework.tests import GangaGPITestCase from GangaTest.Framework.utils import sleep_until_completed class TestTransitions(GangaGPITestCase): def testTransitionsCalled(self): m = MockExeApplication() j = Job(backend=Local())
#if type(self.outputFiles[i]) is not list: # logger.error('need a list of files! %s' % type(self.outputFiles[i])) j.application.outputFiles=self.outputFiles[i] if fin is True: #if type(self.inputFiles[i]) is not list: # logger.error('need a list of files! %s' % type(self.inputFiles[i])) j.application.inputFiles=self.inputFiles[i] if nMacro!=0: j.application.ratMacro=self.ratMacro[i] else: j.application.prodScript=self.prodScript[i] subjobs.append(j) return subjobs ################################################################### from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaSNOplus.Lib.RTHandlers.RTRATProd import RTHandler, WGRTHandler, LCGRTHandler allHandlers.add('RATProd','Local', RTHandler) allHandlers.add('RATProd','PBS', RTHandler) allHandlers.add('RATProd','SGE', RTHandler) allHandlers.add('RATProd','Condor', RTHandler) allHandlers.add('RATProd','LCG', LCGRTHandler) allHandlers.add('RATProd','TestSubmitter', RTHandler) allHandlers.add('RATProd','Interactive', RTHandler) allHandlers.add('RATProd','Batch', RTHandler) allHandlers.add('RATProd','WestGrid', WGRTHandler) logger = Ganga.Utility.logging.getLogger()
jid = "" if job._getRoot().subjobs: jid = job._getRoot().id else: jid = "%d" % job.id environment["OUTPUT_JOBID"] = str(jid) # used for versionning if app.dryrun: environment["DRYRUN"] = "TRUE" if app.dbrelease: environment["ATLASDBREL"] = app.dbrelease inputdata = [] filename = "wrapper.sh" exe = os.path.join(os.path.dirname(__file__), filename) # output sandbox outputbox = jobmasterconfig.outputbox outputbox.extend(subjob_outbox) return StandardJobConfig(File(exe), inputbox, args, outputbox, environment) allHandlers.add('AthenaMC', 'Local', AthenaMCLocalRTHandler) allHandlers.add('AthenaMC', 'LSF', AthenaMCLocalRTHandler) #allHandlers.add('AthenaMC','Condor',AthenaMCLocalRTHandler) # condor has requirements... allHandlers.add('AthenaMC', 'PBS', AthenaMCLocalRTHandler) allHandlers.add('AthenaMC', 'SGE', AthenaMCLocalRTHandler) config = getConfig('AthenaMC') logger = getLogger('AthenaMC')
share_path = os.path.join(share_dir, 'inputdata') if not os.path.isdir(share_path): os.makedirs(share_path) f = open(os.path.join(share_path, 'options_data.pkl'), 'w+b') pickle.dump(inputdata, f) f.close() # store the outputsandbox/outputdata defined in the options file # Can remove this when no-longer need to define outputdata in optsfiles # Can remove the if job: when look into how to do prepare for standalone app # move into RuntimeHandler move whole parsing into options maybe? # try and get the job object # not present if preparing standalone app # must change this as prepare should be seperate from the jpb.inputdata share_path = os.path.join(share_dir, 'output') if not os.path.isdir(share_path): os.makedirs(share_path) f = open(os.path.join(share_path, 'options_parser.pkl'), 'w+b') pickle.dump(parser, f) f.close() #/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\ from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers for (backend, handler) in backend_handlers().iteritems(): allHandlers.add('AppName', backend, handler)
for i, rm in enumerate(self.ratMacro): j = self.createSubjob(job) j.application.ratMacro = rm if self.outputFile != []: j.application.outputFile = self.outputFile[i] if self.inputFile != []: j.application.inputFile = self.inputFile[i] if self.nEvents != []: j.application.nEvents = self.nEvents[i] subjobs.append(j) return subjobs ################################################################### from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaSNOplus.Lib.RTHandlers.RTRATUser import UserRTHandler, UserLCGRTHandler, UserWGRTHandler, UserDiracRTHandler allHandlers.add("RATUser", "Local", UserRTHandler) allHandlers.add("RATUser", "PBS", UserRTHandler) allHandlers.add("RATUser", "SGE", UserRTHandler) allHandlers.add("RATUser", "Condor", UserRTHandler) allHandlers.add("RATUser", "LCG", UserLCGRTHandler) allHandlers.add("RATUser", "Dirac", UserDiracRTHandler) allHandlers.add("RATUser", "TestSubmitter", UserRTHandler) allHandlers.add("RATUser", "Interactive", UserRTHandler) allHandlers.add("RATUser", "Batch", UserRTHandler) allHandlers.add("RATUser", "WestGrid", UserWGRTHandler) logger = Ganga.Utility.logging.getLogger()
def _thread_main(self): """ This is an internal function; the main loop of the background thread """ # Add runtime handlers for all the taskified applications, since now # all the backends are loaded from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from .TaskApplication import handler_map for basename, name in handler_map: for backend in allHandlers.getAllBackends(basename): allHandlers.add(name, backend, allHandlers.get(basename, backend)) from Ganga.Core.GangaRepository import getRegistry while getRegistry("jobs").hasStarted() is not True: time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return while True: from Ganga.Core import monitoring_component if (not monitoring_component is None and monitoring_component.enabled ) or config['ForceTaskMonitoring']: break time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY # and must go away soon for tid in self.ids(): try: self[tid]._getWriteAccess() self[tid].startup() except RegistryError: continue except Exception as err: logger.error( "Unknown/Unexpected Error in starting up tasks main loop") logger.error("Exiting: err=%s" % str(err)) return logger.debug("Entering main loop") # Main loop while self._main_thread is not None and not self._main_thread.should_stop( ): # For each task try to run it if config['ForceTaskMonitoring'] or monitoring_component.enabled: for tid in self.ids(): logger.debug("Running over tid: %s" % str(tid)) try: from Ganga.GPIDev.Lib.Tasks import ITask if isType(self[tid], ITask): # for new ITasks, always need write access self[tid]._getWriteAccess() p = self[tid] else: if self[tid].status in [ "running", "running/pause" ]: self[tid]._getWriteAccess() p = self[tid] elif self[tid].status is 'completed' and ( self[tid].n_status('ready') or self[tid].n_status('running')): self[tid].updateStatus() continue else: continue except RegistryError: # could not acquire lock continue if self._main_thread.should_stop(): break try: from Ganga.GPIDev.Lib.Tasks import ITask if isType(self[tid], ITask): # for new ITasks, always call update() p.update() else: # TODO: Make this user-configurable and add better # error message if (p.n_status("failed") * 100.0 / (20 + p.n_status("completed")) > 20): p.pause() logger.error( "Task %s paused - %i jobs have failed while only %i jobs have completed successfully." % (p.name, p.n_status("failed"), p.n_status("completed"))) logger.error( "Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()" ) logger.error( "You can then continue to run this task with tasks(%i).run()" % p.id) continue numjobs = p.submitJobs() if numjobs > 0: self._flush([p]) # finalise any required transforms p.finaliseTransforms() p.updateStatus() except Exception as x: logger.error( "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x)) type_, value_, traceback_ = sys.exc_info() logger.error("Full traceback:\n %s" % ' '.join( traceback.format_exception(type_, value_, traceback_))) p.pause() if self._main_thread.should_stop(): break if self._main_thread.should_stop(): break logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency'])) # Sleep interruptible for 10 seconds for i in range(0, int(config['TaskLoopFrequency'] * 100)): if self._main_thread.should_stop(): break time.sleep(0.01)
############################################################################### # Ganga Project. http://cern.ch/ganga # # $Id: __init__.py,v 1.1 2008-10-04 17:42:39 karl Exp $ ############################################################################### # File: Vanseg/__init__.py # Author: K. Harrison # Created: 070122 """Package initialisation file""" __author__ = "K.Harrison <*****@*****.**>" __date__ = "22 February 2007" __version__ = "1.0" from Vanseg import Vanseg from VansegLCG import VansegLCG from VansegLocal import VansegLocal from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add( "Vanseg", "Interactive", VansegLocal ) allHandlers.add( "Vanseg", "Local", VansegLocal ) allHandlers.add( "Vanseg", "LCG", VansegLCG )
def submit(self,jobconfig,master_input_sandbox): #if self.error.method == 'submit': self.error.trigger(self._getParent().id) return True def kill(self): #if self.error.method == 'kill': self.error.trigger() return True def updateMonitoringInformation(jobs): for j in jobs: j.backend.error.trigger() updateMonitoringInformation = staticmethod(updateMonitoringInformation) class CrashRuntimeHandler(IRuntimeHandler): def master_prepare(self, app, appmasterconfig): #if appmasterconfig.method == 'master_prepare': appmasterconfig.trigger() return None def prepare(self,app,appsubconfig,appmasterconfig,jobmasterconfig): #if appsubconfig.method == 'prepare': appsubconfig.trigger() return None allHandlers.add('CrashTestApplication','CrashTestBackend',CrashRuntimeHandler)
return super(###CLASS###,self).get_dataset_name() for method in ['getenv','getpack','make','cmt','register','add_output_dir','get_output_dir','add_dataset_name','get_dataset_name']: setattr(eval(method), \"__doc__\", getattr(Gaudi, method).__doc__) """ #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaBoss.Lib.Gaudi.GaudiRunTimeHandler import GaudiRunTimeHandler from GangaBoss.Lib.DIRAC.GaudiDiracRTHandler import GaudiDiracRTHandler for app in available_apps(): exec_str = class_str.replace('###CLASS###', app) if app is 'Vetra': lhcbra = os.path.expandvars("$Vetra_release_area") exec_str = exec_str.replace('###SETLHCBRA###', 'self.lhcb_release_area = "%s"' % lhcbra) else: exec_str = exec_str.replace('###SETLHCBRA###', '') if app is not 'Gaudi': exec(exec_str) for backend in ['LSF','Interactive','PBS','SGE','Local','Condor','Remote']: allHandlers.add(app, backend, GaudiRunTimeHandler) allHandlers.add(app, 'Dirac', GaudiDiracRTHandler) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
return LCGJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env) class gLiteRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.gLite import gLiteJobConfig return gLiteJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Highland', 'LSF', RTHandler) allHandlers.add('Highland', 'Local', RTHandler) allHandlers.add('Highland', 'PBS', RTHandler) allHandlers.add('Highland', 'SGE', RTHandler) allHandlers.add('Highland', 'Condor', RTHandler) allHandlers.add('Highland', 'LCG', LCGRTHandler) allHandlers.add('Highland', 'gLite', gLiteRTHandler) allHandlers.add('Highland', 'TestSubmitter', RTHandler) allHandlers.add('Highland', 'Interactive', RTHandler) allHandlers.add('Highland', 'Batch', RTHandler) allHandlers.add('Highland', 'Cronus', RTHandler) allHandlers.add('Highland', 'Remote', LCGRTHandler) allHandlers.add('Highland', 'CREAM', LCGRTHandler)
# logger.error('need a list of files! %s' % type(self.outputFiles[i])) j.application.outputFiles = self.outputFiles[i] if fin is True: #if type(self.inputFiles[i]) is not list: # logger.error('need a list of files! %s' % type(self.inputFiles[i])) j.application.inputFiles = self.inputFiles[i] if nMacro != 0: j.application.ratMacro = self.ratMacro[i] else: j.application.prodScript = self.prodScript[i] subjobs.append(j) return subjobs ################################################################### from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaSNOplus.Lib.RTHandlers.RTRATProd import RTHandler, WGRTHandler, LCGRTHandler allHandlers.add('RATProd', 'Local', RTHandler) allHandlers.add('RATProd', 'PBS', RTHandler) allHandlers.add('RATProd', 'SGE', RTHandler) allHandlers.add('RATProd', 'Condor', RTHandler) allHandlers.add('RATProd', 'LCG', LCGRTHandler) allHandlers.add('RATProd', 'TestSubmitter', RTHandler) allHandlers.add('RATProd', 'Interactive', RTHandler) allHandlers.add('RATProd', 'Batch', RTHandler) allHandlers.add('RATProd', 'WestGrid', WGRTHandler) logger = Ganga.Utility.logging.getLogger()
# Always check for None OR empty #logger.info("self.module: %s" % str(self.module)) if isType(self.module, str): self.module = File(self.module) if self.module.name == None: raise ApplicationConfigurationError(None, "Application Module not requested") elif self.module.name == "": raise ApplicationConfigurationError(None, "Application Module not requested") else: # Always check we've been given a FILE! self.module.name = fullpath(self.module.name) if not os.path.isfile(self.module.name): msg = 'Module file %s not found.' % self.module.name raise ApplicationConfigurationError(None, msg) def postprocess(self): XMLPostProcessor.postprocess(self, logger) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# # Associate the correct run-time handlers to GaudiPython for various backends. from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler for backend in ['LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote']: allHandlers.add('Bender', backend, LHCbGaudiRunTimeHandler) allHandlers.add('Bender', 'Dirac', LHCbGaudiDiracRunTimeHandler) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
############################################################################### # Ganga Project. http://cern.ch/ganga # # $Id: __init__.py,v 1.1 2009-05-10 16:42:03 karl Exp $ ############################################################################### # File: PDF/__init__.py # Author: K. Harrison # Created: 070122 """Package initialisation file""" __author__ = "K.Harrison <*****@*****.**>" __date__ = "9 May 2009" __version__ = "1.0" from PDF import PDF from PDF_LCG import PDF_LCG from PDF_Local import PDF_Local from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add( "PDF", "Interactive", PDF_Local ) allHandlers.add( "PDF", "Local", PDF_Local ) allHandlers.add( "PDF", "LCG", PDF_LCG )
matchURL = re.search("(http.*://[^/]+)/",Client.baseURLSSL) if matchURL != None: param += " --sourceURL %s " % matchURL.group(1) param += " --trf" jspec.jobParameters = param jspec.metadata="--trf \"%s\"" % arglist #print "SUBJOB DETAILS:",jspec.values() if app.dryrun: print "job.application.dryrun activated, printing out job parameters" print jspec.values() return return jspec from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('AthenaMC','Panda',AthenaMCPandaRTHandler) from Ganga.GPIDev.Credentials import GridProxy gridProxy = GridProxy() from Ganga.Utility.Config import getConfig, ConfigError config = getConfig('AthenaMC') configDQ2 = getConfig('DQ2') from Ganga.Utility.logging import getLogger logger = getLogger()
] ## retrieve the FileStager log if configDQ2['USE_ACCESS_INFO'] or ( job.inputdata and (job.inputdata._name in ['DQ2Dataset', 'AMIDataset', 'EventPicking']) and job.inputdata.type in ['FILE_STAGER']): outputbox += ['FileStager.out', 'FileStager.err'] if job.outputsandbox: outputbox += job.outputsandbox # Switch for DEBUG print-out in logfiles if app.useNoDebugLogs: environment['GANGA_LOG_DEBUG'] = '0' else: environment['GANGA_LOG_DEBUG'] = '1' return LCGJobConfig(File(exe), inputbox, [], outputbox, environment, [], requirements) allHandlers.add('Athena', 'LCG', AthenaLCGRTHandler) allHandlers.add('Athena', 'CREAM', AthenaLCGRTHandler) #allHandlers.add('Athena','Condor',AthenaLCGRTHandler) allHandlers.add('Athena', 'Cronus', AthenaLCGRTHandler) config = getConfig('Athena') configDQ2 = getConfig('DQ2') configSystem = getConfig('System') logger = getLogger()
ifspec.GUID = guid ifspec.fsize = size ifspec.md5sum = checksum ifspec.scope = scope ifspec.dataset = jspec.prodDBlock ifspec.prodDBlock = jspec.prodDBlock ifspec.type = 'input' jspec.addFile(ifspec) if app.input_type: itype = app.input_type else: itype = m.group(5) if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(job.inputdata.names)) else: jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(job.inputdata.names)) # Log files. lfspec = FileSpec() lfspec.lfn = '%s.job.log.tgz' % jspec.jobName lfspec.destinationDBlock = jspec.destinationDBlock lfspec.destinationSE = jspec.destinationSE lfspec.dataset = jspec.destinationDBlock lfspec.type = 'log' jspec.addFile(lfspec) return jspec from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('ProdTrans', 'Panda', ProdTransPandaRTHandler)
if isinstance(exe_cmd, list): if path.isfile(path.abspath(exe_cmd[0])): exe_cmd[0] = path.abspath(exe_cmd[0]) err = None try: rc = subprocess.call(exe_cmd, env=my_env, shell=False) except Exception as x: rc = -9999 print('Exception occured in running process: ' + repr(exe_cmd)) print('Err was: ' + str(x)) subprocess.call('echo $PATH', shell=True) print('PATH: ' + str(my_env['PATH'])) print('PWD: ' + str(my_env['PWD'])) print("files on WN: " + str(listdir('.'))) raise print("files on WN: " + str(listdir('.'))) ###OUTPUTFILESINJECTEDCODE### sys.exit(rc) """ return script_template #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Executable', 'Dirac', ExeDiracRTHandler)
fout = FileSpec() fout.lfn = outfiles[f] fout.type = 'output' fout.dataset = job.outputdata.datasetname fout.destinationDBlock = job.outputdata.datasetname fout.destinationSE = job.backend.site jspec.addFile(fout) param += '-o "%s" ' % (outfiles) # must be double quotes, because python prints strings in 'single quotes' for file in jspec.Files: if file.type in [ 'output', 'log'] and configPanda['chirpconfig']: file.dispatchDBlockToken = configPanda['chirpconfig'] logger.debug('chirp file %s',file) jspec.jobParameters = param return jspec from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('Executable','Panda',ExecutablePandaRTHandler) from Ganga.Utility.Config import getConfig, ConfigError config = getConfig('Athena') configDQ2 = getConfig('DQ2') configPanda = getConfig('Panda') from Ganga.Utility.logging import getLogger logger = getLogger()
return (None,None) config = getConfig('defaults_PandaPilot') #_Properties config.options['exe'].type = type(None) def queueToCE(queue): from pandatools import Client return Client.PandaSites[queue]['queue']+'-'+Client.PandaSites[queue]['localqueue'] def queueToDDM(queue): from pandatools import Client return Client.PandaSites[queue]['ddm'] class LCGRTHandler(IRuntimeHandler): def prepare(self,app,appconfig,appmasterconfig,jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig if not app.queue: raise ApplicationConfigurationError('queue not specified for PandaPilot') args = app.arg_pattern.replace('%%QUEUE%%',app.queue) job = app._getParent() if not job.backend.requirements.sites: job.backend.requirements.sites=[queueToDDM(app.queue)] if not job.backend.requirements.sites: raise ApplicationConfigurationError('Could not map queue name to LCG site') logger.info('Sending pilot for %s to %s'%(app.queue,job.backend.requirements.sites)) return LCGJobConfig(app.exe,app._getParent().inputsandbox,[args],app._getParent().outputsandbox,None) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('PandaPilot','LCG', LCGRTHandler)
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig return LCGJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env) class gLiteRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.gLite import gLiteJobConfig return gLiteJobConfig( app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env ) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add("runND280SandMC", "LSF", RTHandler) allHandlers.add("runND280SandMC", "Local", RTHandler) allHandlers.add("runND280SandMC", "PBS", RTHandler) allHandlers.add("runND280SandMC", "SGE", RTHandler) allHandlers.add("runND280SandMC", "Condor", RTHandler) allHandlers.add("runND280SandMC", "LCG", LCGRTHandler) allHandlers.add("runND280SandMC", "gLite", gLiteRTHandler) allHandlers.add("runND280SandMC", "TestSubmitter", RTHandler) allHandlers.add("runND280SandMC", "Interactive", RTHandler) allHandlers.add("runND280SandMC", "Batch", RTHandler) allHandlers.add("runND280SandMC", "Cronus", RTHandler) allHandlers.add("runND280SandMC", "Remote", LCGRTHandler) allHandlers.add("runND280SandMC", "CREAM", LCGRTHandler)
'no_summary': SimpleItem(defvalue=[], sequence=1, summary_sequence_maxlen=-1, typelist=['str']), 'seq': SimpleItem(defvalue=[], sequence=1, typelist=['int']), 'gList': SimpleItem(defvalue=[], sequence=1, typelist=['str']), 'gListComp': ComponentItem('files', defvalue=[], sequence=1), 'simple_print': SimpleItem(defvalue='', summary_print='_print_summary_simple_print'), 'comp_print': ComponentItem('backends', defvalue=None, summary_print='_print_summary_comp_print'), 'is_prepared': SimpleItem(defvalue=None, strict_sequence=0, visitable=1, copyable=1, hidden=0, typelist=['type(None)', 'bool', ShareDir], protected=0, comparable=1, doc='Location of shared resources. Presence of this attribute implies the application has been prepared.'), }) def configure(self, master_appconfig): return (None, None) def _print_summary_bound_comp(self, value, verbosity_level): return '_print_summary_bound_comp' def _print_summary_bound_simple(self, value, verbosity_level): return '_print_summary_bound_simple' def _print_summary_simple_print(self, value, verbosity_level): return '_print_summary_simple_print' def _print_summary_comp_print(self, value, verbosity_level): return '_print_summary_comp_print' class Handler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): return 0 allHandlers.add('GListApp', 'TestSubmitter', Handler)
raise ApplicationConfigurationError( "Application Module not requested") elif self.module.name == "": raise ApplicationConfigurationError( "Application Module not requested") else: # Always check we've been given a FILE! self.module.name = fullpath(self.module.name) if not os.path.isfile(self.module.name): msg = 'Module file %s not found.' % self.module.name raise ApplicationConfigurationError(msg) def postprocess(self): XMLPostProcessor.postprocess(self, logger) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\# # Associate the correct run-time handlers to GaudiPython for various backends. from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from GangaLHCb.Lib.RTHandlers.LHCbGaudiRunTimeHandler import LHCbGaudiRunTimeHandler from GangaLHCb.Lib.RTHandlers.LHCbGaudiDiracRunTimeHandler import LHCbGaudiDiracRunTimeHandler for backend in [ 'LSF', 'Interactive', 'PBS', 'SGE', 'Local', 'Condor', 'Remote' ]: allHandlers.add('Bender', backend, LHCbGaudiRunTimeHandler) allHandlers.add('Bender', 'Dirac', LHCbGaudiDiracRunTimeHandler) #\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\#
def _thread_main(self): """ This is an internal function; the main loop of the background thread """ # Add runtime handlers for all the taskified applications, since now # all the backends are loaded from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from .TaskApplication import handler_map for basename, name in handler_map: for backend in allHandlers.getAllBackends(basename): allHandlers.add( name, backend, allHandlers.get(basename, backend)) from Ganga.Core.GangaRepository import getRegistry while getRegistry("jobs").hasStarted() is not True: time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return while True: from Ganga.Core import monitoring_component if (not monitoring_component is None and monitoring_component.enabled) or config['ForceTaskMonitoring']: break time.sleep(0.1) if self._main_thread is None or self._main_thread.should_stop(): return # setup the tasks - THIS IS INCOMPATIBLE WITH CONCURRENCY # and must go away soon for tid in self.ids(): try: self[tid]._getWriteAccess() self[tid].startup() except RegistryError: continue except Exception as err: logger.error("Unknown/Unexpected Error in starting up tasks main loop") logger.error("Exiting: err=%s" % str(err)) return logger.debug("Entering main loop") # Main loop while self._main_thread is not None and not self._main_thread.should_stop(): # For each task try to run it if config['ForceTaskMonitoring'] or monitoring_component.enabled: for tid in self.ids(): logger.debug("Running over tid: %s" % str(tid)) try: from Ganga.GPIDev.Lib.Tasks import ITask if isType(self[tid], ITask): # for new ITasks, always need write access self[tid]._getWriteAccess() p = self[tid] else: if self[tid].status in ["running", "running/pause"]: self[tid]._getWriteAccess() p = self[tid] elif self[tid].status is 'completed' and (self[tid].n_status('ready') or self[tid].n_status('running')): self[tid].updateStatus() continue else: continue except RegistryError: # could not acquire lock continue if self._main_thread.should_stop(): break try: from Ganga.GPIDev.Lib.Tasks import ITask if isType(self[tid], ITask): # for new ITasks, always call update() p.update() else: # TODO: Make this user-configurable and add better # error message if (p.n_status("failed") * 100.0 / (20 + p.n_status("completed")) > 20): p.pause() logger.error("Task %s paused - %i jobs have failed while only %i jobs have completed successfully." % ( p.name, p.n_status("failed"), p.n_status("completed"))) logger.error( "Please investigate the cause of the failing jobs and then remove the previously failed jobs using job.remove()") logger.error( "You can then continue to run this task with tasks(%i).run()" % p.id) continue numjobs = p.submitJobs() if numjobs > 0: self._flush([p]) # finalise any required transforms p.finaliseTransforms() p.updateStatus() except Exception as x: logger.error( "Exception occurred in task monitoring loop: %s %s\nThe offending task was paused." % (x.__class__, x)) type_, value_, traceback_ = sys.exc_info() logger.error("Full traceback:\n %s" % ' '.join( traceback.format_exception(type_, value_, traceback_))) p.pause() if self._main_thread.should_stop(): break if self._main_thread.should_stop(): break logger.debug("TaskRegistry Sleeping for: %s seconds" % str(config['TaskLoopFrequency'])) # Sleep interruptible for 10 seconds for i in range(0, int(config['TaskLoopFrequency'] * 100)): if self._main_thread.should_stop(): break time.sleep(0.01)
return LCGJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env) class gLiteRTHandler(IRuntimeHandler): def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.gLite import gLiteJobConfig return gLiteJobConfig(app._scriptname, app._getParent().inputsandbox, [], app._getParent().outputsandbox, app.env) from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers allHandlers.add('oaReconPlusoaAnalysis', 'LSF', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'Local', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'PBS', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'SGE', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'Condor', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'LCG', LCGRTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'gLite', gLiteRTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'TestSubmitter', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'Interactive', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'Batch', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'Cronus', RTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'Remote', LCGRTHandler) allHandlers.add('oaReconPlusoaAnalysis', 'CREAM', LCGRTHandler)
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers from Ganga.GPIDev.Adapters.IRuntimeHandler import IRuntimeHandler class CRABRuntimeHandler(IRuntimeHandler): """Almost empty runtime handler.""" def master_prepare(self, app, appconfig): """No need to prepare this jobs.""" return None def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): """No need to prepare this jobs.""" return None allHandlers.add('CRABApp', 'CRABBackend', CRABRuntimeHandler)