def collectPreparedFiles(app):
"""
Collect the files from the Application in the prepared state
Args:
app (GaudiExec): This expects only the GaudiExec app
"""
if not isinstance(app.is_prepared, ShareDir):
raise ApplicationConfigurationError(
'Failed to prepare Application Correctly')
shared_dir = app.getSharedPath()
input_files, input_folders = [], []
for root, dirs, files in os.walk(shared_dir, topdown=True):
for name in files:
input_files.append(os.path.join(root, name))
for name in dirs:
input_folders.append(os.path.join(root, name))
for file_ in app.getJobObject().inputfiles:
if isinstance(file_, LocalFile):
shutil.copy(
os.path.join(file_.localDir,
os.path.basename(file_.namePattern)), shared_dir)
input_files.append(os.path.join(shared_dir, file_.namePattern))
elif not isinstance(file_, DiracFile):
raise ApplicationConfigurationError(
None,
"File type: %s Not _yet_ supported in GaudiExec" % type(file_))
return input_files, input_folders
def getOptsFiles(self):
"""
This function returns a sanitized absolute path to the self.options file from user input
"""
if self.options:
for this_opt in self.options:
if isinstance(this_opt, LocalFile):
## FIXME LocalFile should return the basename and folder in 2 attibutes so we can piece it together, now it doesn't
full_path = path.join(this_opt.localDir,
this_opt.namePattern)
if not path.exists(full_path):
raise ApplicationConfigurationError(
"Opts File: \'%s\' has been specified but does not exist please check and try again!"
% full_path)
elif isinstance(this_opt, DiracFile):
pass
else:
logger.error("opts: %s" % self.options)
raise ApplicationConfigurationError(
"Opts file type %s not yet supported please contact Ganga devs if you require this support"
% getName(this_opt))
return self.options
else:
raise ApplicationConfigurationError(
"No Opts File has been specified, please provide one!")
def split(self, job):
'''The method creates and returns an array of subjobs starting from the job passed as parameter'''
logger = Ganga.Utility.logging.getLogger()
logger.debug('SBSubmission split called.')
subjobs = []
# check the input mode (dir, none or list)
if job.inputdata.input_mode in ('dir', 'none'):
# in dir or none mode, user has to define the desired number of subjobs
if job.inputdata.number_of_subjobs <= 0:
raise ApplicationConfigurationError(
'You must define the number of subjobs.')
for i in xrange(job.inputdata.number_of_subjobs):
j = self.createSubjob(job)
subjobs.append(j)
elif job.inputdata.input_mode == 'list':
# in list mode user has to define one or more input paths (i.e. one or more files): each list file contains
# a list of path of files that become the input of a job
if len(job.inputdata.input_path) <= 0:
raise ApplicationConfigurationError(
'You must define an input file list.')
# for each list file (i.e. each element in input_path array) a subjob will be created
for f in job.inputdata.input_path:
j = self.createSubjob(job)
j.inputsandbox = [f]
subjobs.append(j)
else:
raise ApplicationConfigurationError('input_mode not recognized.')
logger.debug('%d subjobs created.' % len(subjobs))
return subjobs
def configure(self, masterappconfig):
self.args = convertIntToStringArgs(self.args)
job = self.getJobObject()
if self.cmtsetup == None:
raise ApplicationConfigurationError('No cmt setup script given.')
# Need to handle the possibility of multiple output files !
# setup the output file
for arg in self.args:
if arg == '-o':
raise ApplicationConfigurationError(
'Option "-o" given in args. You must use the outputfile variable instead, even if you have multiple output files.'
)
if self.outputfile == None:
raise ApplicationConfigurationError(
'No output file given. Fill the outputfile variable.')
else:
if type(self.outputfile) == type([]):
for OutFi in self.outputfile:
self.args.append('-o')
self.args.append(OutFi)
else:
self.args.append('-o')
self.args.append(self.outputfile)
# So get the list of filenames get_dataset_filenames() and create a file containing the list of files and put it in the sandbox
if job.inputdata == None:
raise ApplicationConfigurationError(
'The inputdata variable is not defined.')
fileList = job.inputdata.get_dataset_filenames()
if len(fileList) < 1:
raise ApplicationConfigurationError('No input data file given.')
self.args.extend(fileList)
argsStr = ' '.join(self.args)
# Create the bash script and put it in input dir.
script = '#!/bin/bash\n'
script += 'source ' + self.cmtsetup + '\n'
script += self.exe + ' ' + argsStr + '\n'
from Ganga.GPIDev.Lib.File import FileBuffer
if self.exe.find('.exe') > -1:
scriptname = self.exe.replace('.exe', '.sh')
else:
scriptname = self.exe + '.sh'
job.getInputWorkspace().writefile(FileBuffer(scriptname, script),
executable=1)
self._scriptname = job.inputdir + scriptname
return (None, None)
def split(self, job):
logger.debug('StagerJobSplitter called')
if job.inputdata._name != 'StagerDataset':
raise ApplicationConfigurationError(
'StagerJobSplitter requires StagerDataset as input')
if job.application._name not in ['Athena']:
raise ApplicationConfigurationError(
'StagerJobSplitter requires Athena as application')
if self.numfiles <= 0:
self.numfiles = 1
total_evnts = -1
try:
total_evnts = int(job.application.max_events)
except ValueError:
pass
sjob_evnts = -1
subjobs = []
## resolve pfns and guids
pfns = job.inputdata.get_surls()
myguids = pfns.keys()
myguids.sort()
mynames = []
for guid in myguids:
mynames.append(pfns[guid])
if job.inputdata.type in ['', 'DQ2']:
job.inputdata.guids = myguids
job.inputdata.names = mynames
nrjob = int(math.ceil(len(mynames) / float(self.numfiles)))
if total_evnts != -1:
sjob_evnts = int(math.ceil(total_evnts / float(nrjob)))
## split scheme for local jobs: simple splitting
if self.scheme.lower() == 'local':
for i in xrange(0, nrjob):
subjobs.append(
self.__make_subjob__(
job,
myguids[i * self.numfiles:(i + 1) * self.numfiles],
mynames[i * self.numfiles:(i + 1) * self.numfiles],
sjob_evnts))
return subjobs
def configure(self,masterappconfig):
args = convertIntToStringArgs(self.args)
job = self.getJobObject()
if self.cmtsetup == []:
raise ApplicationConfigurationError('No cmt setup script given.')
infiles = job.inputdata.get_dataset_filenames()
if len(infiles) <> 1:
raise ApplicationConfigurationError('Wrong Dataset values')
jn = "%08d" % int(infiles[0])
outConf = ''
outConf += "# Automatically generated config file\n\n"
outConf += "/atmt2k/step/outputFileName corsika_atmpitsim_"+jn+".root\n"
outConf += "/atmt2k/pga/inputFileName "+self.confopts['fluxfile']+"\n"
outConf += "/atmt2k/pga/isOyamaFlux false\n"
outConf += "/atmt2k/pga/inputRandSeed "+str(random.randint(1,9999999))+"\n\n"
outConf += "/run/verbose 0\n"
outConf += "/event/verbose 0\n"
outConf += "/tracking/verbose 0\n"
outConf += "/run/beamOn "+self.confopts['nev']+"\n"
mac = "corsika_atmpitsim_"+jn+".mac"
job.getInputWorkspace().writefile(FileBuffer(mac,outConf),executable=0)
argsStr = ' '.join(args)
# Create the bash script and put it in input dir.
script = '#!/bin/bash\n'
for f in self.cmtsetup:
script += 'source '+f+'\n'
script += 'cd '+job.outputdir+'\n'
script += 'RunAtmPitSim.exe '+argsStr+' '+os.path.join(job.inputdir,mac)+'\n'
script += 'mac=$(dirname $(which RunAtmPitSim.exe))/../app/ntuple_2_nuance.C\n'
script += 'cp $mac .\n'
script += 'root -l -b <<EOF\n'
script += '.L ntuple_2_nuance.C+\n'
script += 'totxt("corsika_atmpitsim_'+jn+'");\n'
script += '.q\n'
script += 'EOF\n'
script += 'mv NoMuons.txt NoMuons_'+jn+'.txt\n'
script += 'mv Config.conf Config_'+jn+'.conf\n'
job.getInputWorkspace().writefile(FileBuffer('runND280.sh',script),executable=1)
self._scriptname = job.inputdir+'runND280.sh'
# Job name given
job.name = jn
return (None,None)
def configure(self, masterappconfig):
self.args = convertIntToStringArgs(self.args)
job = self.getJobObject()
if self.cmtsetup == None:
raise ApplicationConfigurationError('No cmt setup script given.')
# setup the output file
for arg in self.args:
if arg == '-o':
raise ApplicationConfigurationError(
'Option "-o" given in args. You must use the outputfile variable instead.'
)
if self.outputfile == None:
raise ApplicationConfigurationError(
'No output file given. Fill the outputfile variable.')
else:
self.args.append('-o')
self.args.append(self.outputfile)
# So get the list of filenames get_dataset_filenames() and create a file containing the list of files and put it in the sandbox
fileList = job.inputdir + 'FileList'
if not job.inputdata.set_dataset_into_list(fileList):
raise ApplicationConfigurationError(
'Problem with the preparation of the list of input files')
self.args.append(fileList)
argsStr = ' '.join(self.args)
# ANT: Create the bash script here and put it in input dir.
script = '#!/bin/bash\n'
script += 'source ' + self.cmtsetup + '\n'
script += self.exe + ' ' + argsStr + '\n'
from Ganga.GPIDev.Lib.File import FileBuffer
if self.exe.find('.exe') > -1:
scriptname = self.exe.replace('.exe', '.sh')
else:
scriptname = self.exe + '.sh'
job.getInputWorkspace().writefile(FileBuffer(scriptname, script),
executable=1)
self._scriptname = job.inputdir + scriptname
return (None, None)
def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
nonexistentOptFiles = []
for f in self.optsfile:
from Ganga.GPIDev.Lib.File.File import File
if type(f) is str:
myFile = File(f)
else:
myFile = f
myFile.name = fullpath(myFile.name)
if not os.path.isfile(myFile.name):
nonexistentOptFiles.append(myFile)
if len(nonexistentOptFiles):
tmpmsg = "The 'optsfile' attribute contains non-existent file/s: ["
for _f in nonexistentOptFiles:
tmpmsg += "'%s', " % _f.name
msg = tmpmsg[:-2] + ']'
raise ApplicationConfigurationError(msg)
def getNickname(gridProxy=None, allowMissingNickname=True):
import re
from Ganga.Utility.logging import getLogger
from Ganga.GPIDev.Credentials import GridProxy
logger = getLogger()
if not gridProxy:
gridProxy = GridProxy()
nickName = ''
output = gridProxy.info(opt='-all')
for line in output.split('\n'):
if line.startswith('attribute'):
match = re.search('nickname =\s*([^\s]+)\s*\(atlas\)', line)
if match != None:
nickName = match.group(1)
break
# check
if nickName == '':
from Ganga.Core.exceptions import ApplicationConfigurationError
wMessage = 'Could not get nickname from voms proxy. '
wMessage += 'Please register nickname to ATLAS VO via\n\n'
wMessage += ' https://lcg-voms.cern.ch:8443/vo/atlas/vomrs\n'
wMessage += ' [Member Info] -> [Edit Personal Info]\n\n'
wMessage += 'Then you can use the new naming convention "user.nickname" '
wMessage += 'which should be shorter than "userXY.FirstnameLastname".'
if allowMissingNickname:
logger.warning(wMessage)
else:
raise ApplicationConfigurationError(None, wMessage)
return nickName
def getAttributeIndex(attr):
for tmpIdx, tmpAttrName in enumerate(attrNames):
if tmpAttrName.strip() == attr:
return tmpIdx
errStr = "cannot find attribute=%s in %s provided by ELSSI" % (
attr, str(attrNames))
raise ApplicationConfigurationError(None, errStr)
def getGaudiExecInputData(optsfiles, app):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage example:
# Get the data from an options file and assign it to the jobs inputdata field
j.inputdata = getGaudiExecInputData([\"~/cmtuser/DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"], j.application)
This is also called behind the scenes for j.readInputData([\"~/cmtuser/DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
'''
if not isinstance(optsfiles, list):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
raise GangaException("Need a file to parse to call this method")
try:
parser = PythonOptsCmakeParser(optsfiles, app)
except Exception as err:
msg = 'Unable to parse the job options. Please check options files and extraopts.'
logger.error("PythonOptsCmakeParserError:\n%s" % str(err))
raise ApplicationConfigurationError(None, msg)
return parser.get_input_data()
def prepareCommand(app):
"""
Returns the command which is to be run on the worker node
Args:
app (GaudiExec): This expects only the GaudiExec app
"""
all_opts_files = app.getOptsFiles()
opts_names = []
for opts_file in all_opts_files:
if isinstance(opts_file, (LocalFile, DiracFile)):
# Ideally this would NOT need the basename, however LocalFile is special in this regard.
# TODO Fix this after fixing LocalFile
opts_names.append(os.path.basename(opts_file.namePattern))
else:
raise ApplicationConfigurationError(
"The filetype: %s is not yet supported for use as an opts file.\nPlease contact the Ganga devs is you wish this implemented."
% getName(opts_file))
sourceEnv = app.getEnvScript()
if not app.useGaudiRun:
full_cmd = sourceEnv + './run python %s' % app.getWrapperScriptName()
else:
full_cmd = sourceEnv + "./run gaudirun.py %s %s" % (
' '.join(opts_names), GaudiExecDiracRTHandler.data_file)
if app.extraOpts:
full_cmd += ' ' + app.getExtraOptsFileName()
if app.getMetadata:
full_cmd += ' summary.py'
if app.extraArgs:
full_cmd += " " + " ".join(app.extraArgs)
return full_cmd
def diracAPI_script_settings(app):
"""
Set some additional setting on the diracAPI in the JDL making use of any custom parameters set in the backend object
return JDL lines
Args:
app (IApplication): Application belonging to job of interest
"""
job = app.getJobObject()
diracAPI_line = ''
if type(job.backend.settings) is not dict:
raise ApplicationConfigurationError(
'backend.settings should be a dict')
for setting, setting_val in job.backend.settings.iteritems():
if str(setting).startswith('set'):
_setting = str(setting)[3:]
else:
_setting = str(setting)
if type(setting_val) is str:
setting_line = 'j.set###SETTING###("###VALUE###")\n'
else:
setting_line = 'j.set###SETTING###(###VALUE###)\n'
diracAPI_line += setting_line.replace('###SETTING###',
_setting).replace(
'###VALUE###',
str(setting_val))
if diracAPI_line == '':
diracAPI_line = None
return diracAPI_line
def generateDiracInput(app):
"""
Construct a DIRAC input which does not need to be unique to each job but is required to have a unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
"""
input_files, input_folders = collectPreparedFiles(app)
job = app.getJobObject()
if input_folders:
raise ApplicationConfigurationError('Prepared folders not supported yet, please fix this in future')
else:
prep_dir = app.getSharedPath()
addTimestampFile(prep_dir)
prep_file = prep_dir + '.tgz'
tmp_dir = tempfile.gettempdir()
compressed_file = os.path.join(tmp_dir, '__'+os.path.basename(prep_file))
if not job.master:
rjobs = job.subjobs
else:
rjobs = [job]
with tarfile.open(compressed_file, "w:gz") as tar_file:
for name in input_files:
# FIXME Add support for subfiles here once it's working across multiple IGangaFile objects in a consistent way
# Not hacking this in for now just in-case we end up with a mess as a result
tar_file.add(name, arcname=os.path.basename(name))
new_df = uploadLocalFile(job, os.path.basename(compressed_file), tmp_dir)
app.uploadedInput = new_df
def _get_parser(self):
optsfiles = []
for fileitem in self.optsfile:
if type(fileitem) is str:
optsfiles.append(fileitem)
else:
optsfiles.append(fileitem.name)
#optsfiles = [fileitem.name for fileitem in self.optsfile]
# add on XML summary
extraopts = ''
if self.extraopts:
extraopts += self.extraopts
if self.extraopts.find('LHCbApp().XMLSummary') is -1:
extraopts += "\nfrom Gaudi.Configuration import *"
extraopts += "\nfrom Configurables import LHCbApp"
extraopts += "\nLHCbApp().XMLSummary='summary.xml'"
else:
extraopts += "\nfrom Gaudi.Configuration import *"
extraopts += "\nfrom Configurables import LHCbApp"
extraopts += "\nLHCbApp().XMLSummary='summary.xml'"
try:
parser = PythonOptionsParser(optsfiles, extraopts,
self.getenv(False))
except ApplicationConfigurationError as err:
logger.error("PythonOptionsParserError:\n%s" % str(err))
# fix this when preparing not attached to job
msg2 = ''
try:
debug_dir = self.getJobObject().getDebugWorkspace().getPath()
msg2 += 'You can also view this from within ganga '\
'by doing job.peek(\'../debug/gaudirun.<whatever>\').'
except Exception, err:
logger.debug("path Error:\n%s" % str(err))
debug_dir = tempfile.mkdtemp()
messages = err.message.split('###SPLIT###')
if len(messages) is 2:
stdout = open(debug_dir + '/gaudirun.stdout', 'w')
stderr = open(debug_dir + '/gaudirun.stderr', 'w')
stdout.write(messages[0])
stderr.write(messages[1])
stdout.close()
stderr.close()
msg = 'Unable to parse job options! Please check options ' \
'files and extraopts. The output and error streams from gaudirun.py can be ' \
'found in %s and %s respectively . ' % (stdout.name, stderr.name)
else:
f = open(debug_dir + '/gaudirun.out', 'w')
f.write(err.message)
f.close()
msg = 'Unable to parse job options! Please check options ' \
'files and extraopts. The output from gaudirun.py can be ' \
'found in %s . ' % f.name
msg += msg2
logger.debug(msg)
raise ApplicationConfigurationError(msg)
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
# check file is set OK
if not app.script.name:
msg = 'Root.script.name must be set.'
raise ApplicationConfigurationError(msg)
sharedir_scriptpath = os.path.join(get_share_path(app),
os.path.basename(app.script.name))
if not os.path.exists(sharedir_scriptpath):
msg = 'Script must exist!'
raise ApplicationConfigurationError(msg)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))
def _check_inputs(self):
"""Checks the validity of user's entries for Ostap schema"""
if not self.scripts and not self.commands and not self.arguments :
raise ApplicationConfigurationError("Application scripts are not defined")
if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ]
for f in self.scripts : f.name = fullpath ( f.name )
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig):
from Ganga.Lib.LCG import LCGJobConfig
if not app.queue:
raise ApplicationConfigurationError(
None, 'queue not specified for PandaPilot')
args = app.arg_pattern.replace('%%QUEUE%%', app.queue)
job = app._getParent()
if not job.backend.requirements.sites:
job.backend.requirements.sites = [queueToDDM(app.queue)]
if not job.backend.requirements.sites:
raise ApplicationConfigurationError(
None, 'Could not map queue name to LCG site')
logger.info('Sending pilot for %s to %s' %
(app.queue, job.backend.requirements.sites))
return LCGJobConfig(app.exe,
app._getParent().inputsandbox, [args],
app._getParent().outputsandbox, None)
def check(self):
self.initAliases()
if not self.name.replace(".", "").replace(
"_", "").isalnum(): # accept . and _
logger.error(
"Invalid character in task name! Task names are now used for DQ2 datasets; so no spaces, slashes or other special characters are allowed."
)
raise ApplicationConfigurationError(None, "Invalid Task name!")
super(AnaTask, self).check()
def _check_inputs(self):
"""Checks the validity of user's entries for GaudiPython schema"""
# Always check for None OR empty
#logger.info("self.module: %s" % str(self.module))
if isType(self.module, str):
self.module = File(self.module)
if self.module.name == None:
raise ApplicationConfigurationError(
"Application Module not requested")
elif self.module.name == "":
raise ApplicationConfigurationError(
"Application Module not requested")
else:
# Always check we've been given a FILE!
self.module.name = fullpath(self.module.name)
if not os.path.isfile(self.module.name):
msg = 'Module file %s not found.' % self.module.name
raise ApplicationConfigurationError(msg)
def prepare(self, force=False):
"""
This method creates a set of prepared files for the application to pass to the RTHandler
Args:
force (bool): Forces a prepare to be run
"""
if (self.is_prepared is not None) and not force:
raise ApplicationPrepareError(
'%s application has already been prepared. Use prepare(force=True) to prepare again.'
% getName(self))
# lets use the same criteria as the configure() method for checking file existence & sanity
# this will bail us out of prepare if there's somthing odd with the job config - like the executable
# file is unspecified, has a space or is a relative path
self.configure(self)
logger.info('Preparing %s application.' % getName(self))
this_build_target = self.buildGangaTarget()
self.is_prepared = ShareDir()
logger.info('Created shared directory: %s' % (self.is_prepared.name))
try:
# copy any 'preparable' objects into the shared directory
send_to_sharedir = self.copyPreparables()
# add the newly created shared directory into the metadata system
# if the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.copyIntoPrepDir(this_build_target)
all_opts_files = self.getOptsFiles()
for opts_file in all_opts_files:
if isinstance(opts_file, LocalFile):
self.copyIntoPrepDir(
path.join(opts_file.localDir,
path.basename(opts_file.namePattern)))
elif isinstance(opts_file, DiracFile):
# NB safe to put it here as should have expressly setup a path for this job by now.
# We cannot _not_ place this here based upon the backend.
# Always have to put it here regardless of if we're on DIRAC or Local so prepared job can be copied.
opts_file.get(localPath=self.getSharedPath())
else:
raise ApplicationConfigurationError(
"Opts file type %s not yet supported please contact Ganga devs if you require this support"
% getName(opts_file))
self.post_prepare()
except Exception as err:
logger.debug("Err: %s" % str(err))
self.unprepare()
raise
self.cleanGangaTargetArea(this_build_target)
return 1
def split(self, job):
subjobs = []
if not job.inputdata or not isType(job.inputdata, GangaDataset):
raise ApplicationConfigurationError(
"No GangaDataset given for GangaDatasetSplitter")
# find the full file list
full_list = []
for f in job.inputdata.files:
if f.containsWildcards():
# we have a wildcard so grab the subfiles
for sf in f.getSubFiles(process_wildcards=True):
full_list.append(sf)
else:
# no wildcards so just add the file
full_list.append(f)
if len(full_list) == 0:
raise ApplicationConfigurationError(
"GangaDatasetSplitter couldn't find any files to split over")
masterType = type(job.inputdata)
# split based on all the sub files
fid = 0
subjobs = []
filesToRun = len(full_list)
if not self.maxFiles == -1:
filesToRun = min(self.maxFiles, filesToRun)
while fid < filesToRun:
j = self.createSubjob(job)
j.inputdata = masterType()
j.inputdata.treat_as_inputfiles = job.inputdata.treat_as_inputfiles
for sf in full_list[fid:fid + self.files_per_subjob]:
j.inputdata.files.append(sf)
fid += self.files_per_subjob
subjobs.append(j)
return subjobs
def readInputData(self, optsfiles, extraopts=False):
'''Returns a LHCbDataSet object from a list of options files. The
optional argument extraopts will decide if the extraopts string inside
the application is considered or not.
Usage examples:
# Create an LHCbDataset object with the data found in the optionsfile
l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Get the data from an options file and assign it to the jobs inputdata
field
j.inputdata = j.application.readInputData([\"~/cmtuser/\" \
\"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"])
# Assuming you have data in your extraopts, you can use the extraopts.
# In this case your extraopts need to be fully parseable by gaudirun.py
# So you must make sure that you have the proper import statements.
# e.g.
from Gaudi.Configuration import *
# If you mix optionsfiles and extraopts, as usual extraopts may
# overwright your options
#
# Use this to create a new job with data from extraopts of an old job
j=Job(inputdata=jobs[-1].application.readInputData([],True))
'''
def dummyfile():
temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py')
os.write(temp_fd, "Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles) != type([]):
optsfiles = [optsfiles]
# use a dummy file to keep the parser happy
if len(optsfiles) == 0:
optsfiles.append(dummyfile())
if extraopts:
extraopts = self.extraopts
else:
extraopts = ""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(optsfiles, extraopts,
self.getenv(False))
except Exception as err:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
logger.error("PythonOptionsParserError:\n%s" % str(err))
raise ApplicationConfigurationError(msg)
return GPIProxyObjectFactory(parser.get_input_data())
def getID(self):
"""Return the index of this trf in the parent task"""
# if the id isn't already set, use the index from the parent Task
if self.id < 0:
task = self._getParent()
if not task:
raise ApplicationConfigurationError(
None, "This transform has not been associated with a task and so there is no ID available")
self.id = task.transforms.index(self)
return self.id
def loaddriver():
"""Create new driver based on Robot configuration options.
Example of relevant configuration options:
[Robot]
Driver_Run = ['submit', 30, 'extract', 'report']
Driver_Repeat = False
Driver_Action_submit = GangaRobot.Lib.Core.CoreSubmitter.CoreSubmitter
Driver_Action_extract = GangaRobot.Lib.Core.CoreExtractor.CoreExtractor
Driver_Action_report = GangaRobot.Lib.Core.CoreReporter.CoreReporter
"""
KEY_RUN = 'Driver_Run'
KEY_REPEAT = 'Driver_Repeat'
KEY_ACTION_PREFIX = 'Driver_Action_'
config = Utility.getconfig()
run = config[KEY_RUN]
repeat = config[KEY_REPEAT]
actions = {}
#load action classes
for key in config:
if key.startswith(KEY_ACTION_PREFIX):
action = key[len(KEY_ACTION_PREFIX):]
fqcn = config[key]
try:
actions[action] = _loadclass(fqcn)
except Exception as e:
raise ApplicationConfigurationError(
"Cannot load class '%s'. Exception: '%s'" % (fqcn, e))
#check actions exist for run
for action in run:
if not action in actions:
try:
int(action)
except ValueError as e:
raise ApplicationConfigurationError(
"Unknown action '%s'. Exception: '%s'" % (action, e))
return Driver(run=run, actions=actions, repeat=repeat)
def getID(self):
"""Get the ID of this unit within the transform"""
# if the id isn't already set, use the index from the parent Task
if self.id < 0:
trf = self._getParent()
if not trf:
raise ApplicationConfigurationError(
"This unit has not been associated with a transform and so there is no ID available"
)
self.id = trf.units.index(self)
return self.id
def check(self):
"""Check this transform has valid data, etc. and has the correct units"""
# ignore anything but new transforms
if self.status != "new":
return
# first, validate the transform
if not self.validate():
raise ApplicationConfigurationError(
None, "Validate failed for Transform %s" % self.name)
self.updateStatus("running")
def loadPlugins(c):
import sys
from Ganga.Utility.logging import getLogger
try:
import Lib.Panda
import Lib.Jedi
except SystemExit:
from Ganga.Core.exceptions import ApplicationConfigurationError
import commands
(s, o) = commands.getstatusoutput('curl --version')
if (s):
raise ApplicationConfigurationError(
"Couldn't load Panda Client: ensure 'curl' is available")
else:
raise ApplicationConfigurationError("Couldn't load Panda Client")
import Lib.Athena
import Lib.Executable
import Lib.ProdTrans
return None
def getDBDatasets(jobO, trf, dbrelease):
from pandatools import Client
# get DB datasets
dbrFiles = {}
dbrDsList = []
if trf or dbrelease != '':
if trf:
# parse jobO for TRF
tmpItems = jobO.split()
else:
# mimic a trf parameter to reuse following algorithm
tmpItems = ['%DB=' + dbrelease]
# look for DBRelease
for tmpItem in tmpItems:
match = re.search('%DB=([^:]+):(.+)$', tmpItem)
if match:
tmpDbrDS = match.group(1)
tmpDbrLFN = match.group(2)
# get files in the dataset
if not tmpDbrDS in dbrDsList:
logger.info("Querying files in dataset:%s" % tmpDbrDS)
try:
tmpList = Client.queryFilesInDataset(tmpDbrDS, False)
except:
raise ApplicationConfigurationError(
None,
"ERROR : error while looking up dataset %s. Perhaps this dataset does not exist?"
% tmpDbrDS)
# append
for tmpLFN, tmpVal in tmpList.iteritems():
dbrFiles[tmpLFN] = tmpVal
dbrDsList.append(tmpDbrDS)
# check
if tmpDbrLFN not in dbrFiles:
raise ApplicationConfigurationError(
None,
"ERROR : %s is not in %s" % (tmpDbrLFN, tmpDbrDS))
return dbrFiles, dbrDsList
def master_prepare(self, app, appmasterconfig):
inputsandbox, outputsandbox = master_sandbox_prepare(
app, appmasterconfig)
if type(app.exe) == File:
input_dir = app.getJobObject().getInputWorkspace().getPath()
exefile = os.path.join(input_dir, os.path.basename(app.exe.name))
if not os.path.exists(exefile):
msg = 'Executable: "%s" must exist!' % str(exefile)
raise ApplicationConfigurationError(None, msg)
os.system('chmod +x %s' % exefile)
return StandardJobConfig(inputbox=unique(inputsandbox),
outputbox=unique(outputsandbox))