def collectPreparedFiles(app): """ Collect the files from the Application in the prepared state Args: app (GaudiExec): This expects only the GaudiExec app """ if not isinstance(app.is_prepared, ShareDir): raise ApplicationConfigurationError( 'Failed to prepare Application Correctly') shared_dir = app.getSharedPath() input_files, input_folders = [], [] for root, dirs, files in os.walk(shared_dir, topdown=True): for name in files: input_files.append(os.path.join(root, name)) for name in dirs: input_folders.append(os.path.join(root, name)) for file_ in app.getJobObject().inputfiles: if isinstance(file_, LocalFile): shutil.copy( os.path.join(file_.localDir, os.path.basename(file_.namePattern)), shared_dir) input_files.append(os.path.join(shared_dir, file_.namePattern)) elif not isinstance(file_, DiracFile): raise ApplicationConfigurationError( None, "File type: %s Not _yet_ supported in GaudiExec" % type(file_)) return input_files, input_folders
def getOptsFiles(self): """ This function returns a sanitized absolute path to the self.options file from user input """ if self.options: for this_opt in self.options: if isinstance(this_opt, LocalFile): ## FIXME LocalFile should return the basename and folder in 2 attibutes so we can piece it together, now it doesn't full_path = path.join(this_opt.localDir, this_opt.namePattern) if not path.exists(full_path): raise ApplicationConfigurationError( "Opts File: \'%s\' has been specified but does not exist please check and try again!" % full_path) elif isinstance(this_opt, DiracFile): pass else: logger.error("opts: %s" % self.options) raise ApplicationConfigurationError( "Opts file type %s not yet supported please contact Ganga devs if you require this support" % getName(this_opt)) return self.options else: raise ApplicationConfigurationError( "No Opts File has been specified, please provide one!")
def split(self, job): '''The method creates and returns an array of subjobs starting from the job passed as parameter''' logger = Ganga.Utility.logging.getLogger() logger.debug('SBSubmission split called.') subjobs = [] # check the input mode (dir, none or list) if job.inputdata.input_mode in ('dir', 'none'): # in dir or none mode, user has to define the desired number of subjobs if job.inputdata.number_of_subjobs <= 0: raise ApplicationConfigurationError( 'You must define the number of subjobs.') for i in xrange(job.inputdata.number_of_subjobs): j = self.createSubjob(job) subjobs.append(j) elif job.inputdata.input_mode == 'list': # in list mode user has to define one or more input paths (i.e. one or more files): each list file contains # a list of path of files that become the input of a job if len(job.inputdata.input_path) <= 0: raise ApplicationConfigurationError( 'You must define an input file list.') # for each list file (i.e. each element in input_path array) a subjob will be created for f in job.inputdata.input_path: j = self.createSubjob(job) j.inputsandbox = [f] subjobs.append(j) else: raise ApplicationConfigurationError('input_mode not recognized.') logger.debug('%d subjobs created.' % len(subjobs)) return subjobs
def configure(self, masterappconfig): self.args = convertIntToStringArgs(self.args) job = self.getJobObject() if self.cmtsetup == None: raise ApplicationConfigurationError('No cmt setup script given.') # Need to handle the possibility of multiple output files ! # setup the output file for arg in self.args: if arg == '-o': raise ApplicationConfigurationError( 'Option "-o" given in args. You must use the outputfile variable instead, even if you have multiple output files.' ) if self.outputfile == None: raise ApplicationConfigurationError( 'No output file given. Fill the outputfile variable.') else: if type(self.outputfile) == type([]): for OutFi in self.outputfile: self.args.append('-o') self.args.append(OutFi) else: self.args.append('-o') self.args.append(self.outputfile) # So get the list of filenames get_dataset_filenames() and create a file containing the list of files and put it in the sandbox if job.inputdata == None: raise ApplicationConfigurationError( 'The inputdata variable is not defined.') fileList = job.inputdata.get_dataset_filenames() if len(fileList) < 1: raise ApplicationConfigurationError('No input data file given.') self.args.extend(fileList) argsStr = ' '.join(self.args) # Create the bash script and put it in input dir. script = '#!/bin/bash\n' script += 'source ' + self.cmtsetup + '\n' script += self.exe + ' ' + argsStr + '\n' from Ganga.GPIDev.Lib.File import FileBuffer if self.exe.find('.exe') > -1: scriptname = self.exe.replace('.exe', '.sh') else: scriptname = self.exe + '.sh' job.getInputWorkspace().writefile(FileBuffer(scriptname, script), executable=1) self._scriptname = job.inputdir + scriptname return (None, None)
def split(self, job): logger.debug('StagerJobSplitter called') if job.inputdata._name != 'StagerDataset': raise ApplicationConfigurationError( 'StagerJobSplitter requires StagerDataset as input') if job.application._name not in ['Athena']: raise ApplicationConfigurationError( 'StagerJobSplitter requires Athena as application') if self.numfiles <= 0: self.numfiles = 1 total_evnts = -1 try: total_evnts = int(job.application.max_events) except ValueError: pass sjob_evnts = -1 subjobs = [] ## resolve pfns and guids pfns = job.inputdata.get_surls() myguids = pfns.keys() myguids.sort() mynames = [] for guid in myguids: mynames.append(pfns[guid]) if job.inputdata.type in ['', 'DQ2']: job.inputdata.guids = myguids job.inputdata.names = mynames nrjob = int(math.ceil(len(mynames) / float(self.numfiles))) if total_evnts != -1: sjob_evnts = int(math.ceil(total_evnts / float(nrjob))) ## split scheme for local jobs: simple splitting if self.scheme.lower() == 'local': for i in xrange(0, nrjob): subjobs.append( self.__make_subjob__( job, myguids[i * self.numfiles:(i + 1) * self.numfiles], mynames[i * self.numfiles:(i + 1) * self.numfiles], sjob_evnts)) return subjobs
def configure(self,masterappconfig): args = convertIntToStringArgs(self.args) job = self.getJobObject() if self.cmtsetup == []: raise ApplicationConfigurationError('No cmt setup script given.') infiles = job.inputdata.get_dataset_filenames() if len(infiles) <> 1: raise ApplicationConfigurationError('Wrong Dataset values') jn = "%08d" % int(infiles[0]) outConf = '' outConf += "# Automatically generated config file\n\n" outConf += "/atmt2k/step/outputFileName corsika_atmpitsim_"+jn+".root\n" outConf += "/atmt2k/pga/inputFileName "+self.confopts['fluxfile']+"\n" outConf += "/atmt2k/pga/isOyamaFlux false\n" outConf += "/atmt2k/pga/inputRandSeed "+str(random.randint(1,9999999))+"\n\n" outConf += "/run/verbose 0\n" outConf += "/event/verbose 0\n" outConf += "/tracking/verbose 0\n" outConf += "/run/beamOn "+self.confopts['nev']+"\n" mac = "corsika_atmpitsim_"+jn+".mac" job.getInputWorkspace().writefile(FileBuffer(mac,outConf),executable=0) argsStr = ' '.join(args) # Create the bash script and put it in input dir. script = '#!/bin/bash\n' for f in self.cmtsetup: script += 'source '+f+'\n' script += 'cd '+job.outputdir+'\n' script += 'RunAtmPitSim.exe '+argsStr+' '+os.path.join(job.inputdir,mac)+'\n' script += 'mac=$(dirname $(which RunAtmPitSim.exe))/../app/ntuple_2_nuance.C\n' script += 'cp $mac .\n' script += 'root -l -b <<EOF\n' script += '.L ntuple_2_nuance.C+\n' script += 'totxt("corsika_atmpitsim_'+jn+'");\n' script += '.q\n' script += 'EOF\n' script += 'mv NoMuons.txt NoMuons_'+jn+'.txt\n' script += 'mv Config.conf Config_'+jn+'.conf\n' job.getInputWorkspace().writefile(FileBuffer('runND280.sh',script),executable=1) self._scriptname = job.inputdir+'runND280.sh' # Job name given job.name = jn return (None,None)
def configure(self, masterappconfig): self.args = convertIntToStringArgs(self.args) job = self.getJobObject() if self.cmtsetup == None: raise ApplicationConfigurationError('No cmt setup script given.') # setup the output file for arg in self.args: if arg == '-o': raise ApplicationConfigurationError( 'Option "-o" given in args. You must use the outputfile variable instead.' ) if self.outputfile == None: raise ApplicationConfigurationError( 'No output file given. Fill the outputfile variable.') else: self.args.append('-o') self.args.append(self.outputfile) # So get the list of filenames get_dataset_filenames() and create a file containing the list of files and put it in the sandbox fileList = job.inputdir + 'FileList' if not job.inputdata.set_dataset_into_list(fileList): raise ApplicationConfigurationError( 'Problem with the preparation of the list of input files') self.args.append(fileList) argsStr = ' '.join(self.args) # ANT: Create the bash script here and put it in input dir. script = '#!/bin/bash\n' script += 'source ' + self.cmtsetup + '\n' script += self.exe + ' ' + argsStr + '\n' from Ganga.GPIDev.Lib.File import FileBuffer if self.exe.find('.exe') > -1: scriptname = self.exe.replace('.exe', '.sh') else: scriptname = self.exe + '.sh' job.getInputWorkspace().writefile(FileBuffer(scriptname, script), executable=1) self._scriptname = job.inputdir + scriptname return (None, None)
def _check_inputs(self): """Checks the validity of some of user's entries for Gaudi schema""" # Warn user that no optsfiles given if len(self.optsfile) == 0: logger.warning("The 'optsfile' is not set. I hope this is OK!") else: # Check for non-exting optsfiles defined nonexistentOptFiles = [] for f in self.optsfile: from Ganga.GPIDev.Lib.File.File import File if type(f) is str: myFile = File(f) else: myFile = f myFile.name = fullpath(myFile.name) if not os.path.isfile(myFile.name): nonexistentOptFiles.append(myFile) if len(nonexistentOptFiles): tmpmsg = "The 'optsfile' attribute contains non-existent file/s: [" for _f in nonexistentOptFiles: tmpmsg += "'%s', " % _f.name msg = tmpmsg[:-2] + ']' raise ApplicationConfigurationError(msg)
def getNickname(gridProxy=None, allowMissingNickname=True): import re from Ganga.Utility.logging import getLogger from Ganga.GPIDev.Credentials import GridProxy logger = getLogger() if not gridProxy: gridProxy = GridProxy() nickName = '' output = gridProxy.info(opt='-all') for line in output.split('\n'): if line.startswith('attribute'): match = re.search('nickname =\s*([^\s]+)\s*\(atlas\)', line) if match != None: nickName = match.group(1) break # check if nickName == '': from Ganga.Core.exceptions import ApplicationConfigurationError wMessage = 'Could not get nickname from voms proxy. ' wMessage += 'Please register nickname to ATLAS VO via\n\n' wMessage += ' https://lcg-voms.cern.ch:8443/vo/atlas/vomrs\n' wMessage += ' [Member Info] -> [Edit Personal Info]\n\n' wMessage += 'Then you can use the new naming convention "user.nickname" ' wMessage += 'which should be shorter than "userXY.FirstnameLastname".' if allowMissingNickname: logger.warning(wMessage) else: raise ApplicationConfigurationError(None, wMessage) return nickName
def getAttributeIndex(attr): for tmpIdx, tmpAttrName in enumerate(attrNames): if tmpAttrName.strip() == attr: return tmpIdx errStr = "cannot find attribute=%s in %s provided by ELSSI" % ( attr, str(attrNames)) raise ApplicationConfigurationError(None, errStr)
def getGaudiExecInputData(optsfiles, app): '''Returns a LHCbDataSet object from a list of options files. The optional argument extraopts will decide if the extraopts string inside the application is considered or not. Usage example: # Get the data from an options file and assign it to the jobs inputdata field j.inputdata = getGaudiExecInputData([\"~/cmtuser/DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"], j.application) This is also called behind the scenes for j.readInputData([\"~/cmtuser/DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) ''' if not isinstance(optsfiles, list): optsfiles = [optsfiles] # use a dummy file to keep the parser happy if len(optsfiles) == 0: raise GangaException("Need a file to parse to call this method") try: parser = PythonOptsCmakeParser(optsfiles, app) except Exception as err: msg = 'Unable to parse the job options. Please check options files and extraopts.' logger.error("PythonOptsCmakeParserError:\n%s" % str(err)) raise ApplicationConfigurationError(None, msg) return parser.get_input_data()
def prepareCommand(app): """ Returns the command which is to be run on the worker node Args: app (GaudiExec): This expects only the GaudiExec app """ all_opts_files = app.getOptsFiles() opts_names = [] for opts_file in all_opts_files: if isinstance(opts_file, (LocalFile, DiracFile)): # Ideally this would NOT need the basename, however LocalFile is special in this regard. # TODO Fix this after fixing LocalFile opts_names.append(os.path.basename(opts_file.namePattern)) else: raise ApplicationConfigurationError( "The filetype: %s is not yet supported for use as an opts file.\nPlease contact the Ganga devs is you wish this implemented." % getName(opts_file)) sourceEnv = app.getEnvScript() if not app.useGaudiRun: full_cmd = sourceEnv + './run python %s' % app.getWrapperScriptName() else: full_cmd = sourceEnv + "./run gaudirun.py %s %s" % ( ' '.join(opts_names), GaudiExecDiracRTHandler.data_file) if app.extraOpts: full_cmd += ' ' + app.getExtraOptsFileName() if app.getMetadata: full_cmd += ' summary.py' if app.extraArgs: full_cmd += " " + " ".join(app.extraArgs) return full_cmd
def diracAPI_script_settings(app): """ Set some additional setting on the diracAPI in the JDL making use of any custom parameters set in the backend object return JDL lines Args: app (IApplication): Application belonging to job of interest """ job = app.getJobObject() diracAPI_line = '' if type(job.backend.settings) is not dict: raise ApplicationConfigurationError( 'backend.settings should be a dict') for setting, setting_val in job.backend.settings.iteritems(): if str(setting).startswith('set'): _setting = str(setting)[3:] else: _setting = str(setting) if type(setting_val) is str: setting_line = 'j.set###SETTING###("###VALUE###")\n' else: setting_line = 'j.set###SETTING###(###VALUE###)\n' diracAPI_line += setting_line.replace('###SETTING###', _setting).replace( '###VALUE###', str(setting_val)) if diracAPI_line == '': diracAPI_line = None return diracAPI_line
def generateDiracInput(app): """ Construct a DIRAC input which does not need to be unique to each job but is required to have a unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed """ input_files, input_folders = collectPreparedFiles(app) job = app.getJobObject() if input_folders: raise ApplicationConfigurationError('Prepared folders not supported yet, please fix this in future') else: prep_dir = app.getSharedPath() addTimestampFile(prep_dir) prep_file = prep_dir + '.tgz' tmp_dir = tempfile.gettempdir() compressed_file = os.path.join(tmp_dir, '__'+os.path.basename(prep_file)) if not job.master: rjobs = job.subjobs else: rjobs = [job] with tarfile.open(compressed_file, "w:gz") as tar_file: for name in input_files: # FIXME Add support for subfiles here once it's working across multiple IGangaFile objects in a consistent way # Not hacking this in for now just in-case we end up with a mess as a result tar_file.add(name, arcname=os.path.basename(name)) new_df = uploadLocalFile(job, os.path.basename(compressed_file), tmp_dir) app.uploadedInput = new_df
def _get_parser(self): optsfiles = [] for fileitem in self.optsfile: if type(fileitem) is str: optsfiles.append(fileitem) else: optsfiles.append(fileitem.name) #optsfiles = [fileitem.name for fileitem in self.optsfile] # add on XML summary extraopts = '' if self.extraopts: extraopts += self.extraopts if self.extraopts.find('LHCbApp().XMLSummary') is -1: extraopts += "\nfrom Gaudi.Configuration import *" extraopts += "\nfrom Configurables import LHCbApp" extraopts += "\nLHCbApp().XMLSummary='summary.xml'" else: extraopts += "\nfrom Gaudi.Configuration import *" extraopts += "\nfrom Configurables import LHCbApp" extraopts += "\nLHCbApp().XMLSummary='summary.xml'" try: parser = PythonOptionsParser(optsfiles, extraopts, self.getenv(False)) except ApplicationConfigurationError as err: logger.error("PythonOptionsParserError:\n%s" % str(err)) # fix this when preparing not attached to job msg2 = '' try: debug_dir = self.getJobObject().getDebugWorkspace().getPath() msg2 += 'You can also view this from within ganga '\ 'by doing job.peek(\'../debug/gaudirun.<whatever>\').' except Exception, err: logger.debug("path Error:\n%s" % str(err)) debug_dir = tempfile.mkdtemp() messages = err.message.split('###SPLIT###') if len(messages) is 2: stdout = open(debug_dir + '/gaudirun.stdout', 'w') stderr = open(debug_dir + '/gaudirun.stderr', 'w') stdout.write(messages[0]) stderr.write(messages[1]) stdout.close() stderr.close() msg = 'Unable to parse job options! Please check options ' \ 'files and extraopts. The output and error streams from gaudirun.py can be ' \ 'found in %s and %s respectively . ' % (stdout.name, stderr.name) else: f = open(debug_dir + '/gaudirun.out', 'w') f.write(err.message) f.close() msg = 'Unable to parse job options! Please check options ' \ 'files and extraopts. The output from gaudirun.py can be ' \ 'found in %s . ' % f.name msg += msg2 logger.debug(msg) raise ApplicationConfigurationError(msg)
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) # check file is set OK if not app.script.name: msg = 'Root.script.name must be set.' raise ApplicationConfigurationError(msg) sharedir_scriptpath = os.path.join(get_share_path(app), os.path.basename(app.script.name)) if not os.path.exists(sharedir_scriptpath): msg = 'Script must exist!' raise ApplicationConfigurationError(msg) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def _check_inputs(self): """Checks the validity of user's entries for Ostap schema""" if not self.scripts and not self.commands and not self.arguments : raise ApplicationConfigurationError("Application scripts are not defined") if isinstance ( self.scripts , str ) : self.scripts = [ File ( self.scripts ) ] for f in self.scripts : f.name = fullpath ( f.name )
def prepare(self, app, appconfig, appmasterconfig, jobmasterconfig): from Ganga.Lib.LCG import LCGJobConfig if not app.queue: raise ApplicationConfigurationError( None, 'queue not specified for PandaPilot') args = app.arg_pattern.replace('%%QUEUE%%', app.queue) job = app._getParent() if not job.backend.requirements.sites: job.backend.requirements.sites = [queueToDDM(app.queue)] if not job.backend.requirements.sites: raise ApplicationConfigurationError( None, 'Could not map queue name to LCG site') logger.info('Sending pilot for %s to %s' % (app.queue, job.backend.requirements.sites)) return LCGJobConfig(app.exe, app._getParent().inputsandbox, [args], app._getParent().outputsandbox, None)
def check(self): self.initAliases() if not self.name.replace(".", "").replace( "_", "").isalnum(): # accept . and _ logger.error( "Invalid character in task name! Task names are now used for DQ2 datasets; so no spaces, slashes or other special characters are allowed." ) raise ApplicationConfigurationError(None, "Invalid Task name!") super(AnaTask, self).check()
def _check_inputs(self): """Checks the validity of user's entries for GaudiPython schema""" # Always check for None OR empty #logger.info("self.module: %s" % str(self.module)) if isType(self.module, str): self.module = File(self.module) if self.module.name == None: raise ApplicationConfigurationError( "Application Module not requested") elif self.module.name == "": raise ApplicationConfigurationError( "Application Module not requested") else: # Always check we've been given a FILE! self.module.name = fullpath(self.module.name) if not os.path.isfile(self.module.name): msg = 'Module file %s not found.' % self.module.name raise ApplicationConfigurationError(msg)
def prepare(self, force=False): """ This method creates a set of prepared files for the application to pass to the RTHandler Args: force (bool): Forces a prepare to be run """ if (self.is_prepared is not None) and not force: raise ApplicationPrepareError( '%s application has already been prepared. Use prepare(force=True) to prepare again.' % getName(self)) # lets use the same criteria as the configure() method for checking file existence & sanity # this will bail us out of prepare if there's somthing odd with the job config - like the executable # file is unspecified, has a space or is a relative path self.configure(self) logger.info('Preparing %s application.' % getName(self)) this_build_target = self.buildGangaTarget() self.is_prepared = ShareDir() logger.info('Created shared directory: %s' % (self.is_prepared.name)) try: # copy any 'preparable' objects into the shared directory send_to_sharedir = self.copyPreparables() # add the newly created shared directory into the metadata system # if the app is associated with a persisted object self.checkPreparedHasParent(self) self.copyIntoPrepDir(this_build_target) all_opts_files = self.getOptsFiles() for opts_file in all_opts_files: if isinstance(opts_file, LocalFile): self.copyIntoPrepDir( path.join(opts_file.localDir, path.basename(opts_file.namePattern))) elif isinstance(opts_file, DiracFile): # NB safe to put it here as should have expressly setup a path for this job by now. # We cannot _not_ place this here based upon the backend. # Always have to put it here regardless of if we're on DIRAC or Local so prepared job can be copied. opts_file.get(localPath=self.getSharedPath()) else: raise ApplicationConfigurationError( "Opts file type %s not yet supported please contact Ganga devs if you require this support" % getName(opts_file)) self.post_prepare() except Exception as err: logger.debug("Err: %s" % str(err)) self.unprepare() raise self.cleanGangaTargetArea(this_build_target) return 1
def split(self, job): subjobs = [] if not job.inputdata or not isType(job.inputdata, GangaDataset): raise ApplicationConfigurationError( "No GangaDataset given for GangaDatasetSplitter") # find the full file list full_list = [] for f in job.inputdata.files: if f.containsWildcards(): # we have a wildcard so grab the subfiles for sf in f.getSubFiles(process_wildcards=True): full_list.append(sf) else: # no wildcards so just add the file full_list.append(f) if len(full_list) == 0: raise ApplicationConfigurationError( "GangaDatasetSplitter couldn't find any files to split over") masterType = type(job.inputdata) # split based on all the sub files fid = 0 subjobs = [] filesToRun = len(full_list) if not self.maxFiles == -1: filesToRun = min(self.maxFiles, filesToRun) while fid < filesToRun: j = self.createSubjob(job) j.inputdata = masterType() j.inputdata.treat_as_inputfiles = job.inputdata.treat_as_inputfiles for sf in full_list[fid:fid + self.files_per_subjob]: j.inputdata.files.append(sf) fid += self.files_per_subjob subjobs.append(j) return subjobs
def readInputData(self, optsfiles, extraopts=False): '''Returns a LHCbDataSet object from a list of options files. The optional argument extraopts will decide if the extraopts string inside the application is considered or not. Usage examples: # Create an LHCbDataset object with the data found in the optionsfile l=DaVinci(version='v22r0p2').readInputData([\"~/cmtuser/\" \ \"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) # Get the data from an options file and assign it to the jobs inputdata field j.inputdata = j.application.readInputData([\"~/cmtuser/\" \ \"DaVinci_v22r0p2/Tutorial/Analysis/options/Bs2JpsiPhi2008.py\"]) # Assuming you have data in your extraopts, you can use the extraopts. # In this case your extraopts need to be fully parseable by gaudirun.py # So you must make sure that you have the proper import statements. # e.g. from Gaudi.Configuration import * # If you mix optionsfiles and extraopts, as usual extraopts may # overwright your options # # Use this to create a new job with data from extraopts of an old job j=Job(inputdata=jobs[-1].application.readInputData([],True)) ''' def dummyfile(): temp_fd, temp_filename = tempfile.mkstemp(text=True, suffix='.py') os.write(temp_fd, "Dummy file to keep the Optionsparser happy") os.close(temp_fd) return temp_filename if type(optsfiles) != type([]): optsfiles = [optsfiles] # use a dummy file to keep the parser happy if len(optsfiles) == 0: optsfiles.append(dummyfile()) if extraopts: extraopts = self.extraopts else: extraopts = "" # parser = check_inputs(optsfiles, extraopts, self.env) try: parser = PythonOptionsParser(optsfiles, extraopts, self.getenv(False)) except Exception as err: msg = 'Unable to parse the job options. Please check options ' \ 'files and extraopts.' logger.error("PythonOptionsParserError:\n%s" % str(err)) raise ApplicationConfigurationError(msg) return GPIProxyObjectFactory(parser.get_input_data())
def getID(self): """Return the index of this trf in the parent task""" # if the id isn't already set, use the index from the parent Task if self.id < 0: task = self._getParent() if not task: raise ApplicationConfigurationError( None, "This transform has not been associated with a task and so there is no ID available") self.id = task.transforms.index(self) return self.id
def loaddriver(): """Create new driver based on Robot configuration options. Example of relevant configuration options: [Robot] Driver_Run = ['submit', 30, 'extract', 'report'] Driver_Repeat = False Driver_Action_submit = GangaRobot.Lib.Core.CoreSubmitter.CoreSubmitter Driver_Action_extract = GangaRobot.Lib.Core.CoreExtractor.CoreExtractor Driver_Action_report = GangaRobot.Lib.Core.CoreReporter.CoreReporter """ KEY_RUN = 'Driver_Run' KEY_REPEAT = 'Driver_Repeat' KEY_ACTION_PREFIX = 'Driver_Action_' config = Utility.getconfig() run = config[KEY_RUN] repeat = config[KEY_REPEAT] actions = {} #load action classes for key in config: if key.startswith(KEY_ACTION_PREFIX): action = key[len(KEY_ACTION_PREFIX):] fqcn = config[key] try: actions[action] = _loadclass(fqcn) except Exception as e: raise ApplicationConfigurationError( "Cannot load class '%s'. Exception: '%s'" % (fqcn, e)) #check actions exist for run for action in run: if not action in actions: try: int(action) except ValueError as e: raise ApplicationConfigurationError( "Unknown action '%s'. Exception: '%s'" % (action, e)) return Driver(run=run, actions=actions, repeat=repeat)
def getID(self): """Get the ID of this unit within the transform""" # if the id isn't already set, use the index from the parent Task if self.id < 0: trf = self._getParent() if not trf: raise ApplicationConfigurationError( "This unit has not been associated with a transform and so there is no ID available" ) self.id = trf.units.index(self) return self.id
def check(self): """Check this transform has valid data, etc. and has the correct units""" # ignore anything but new transforms if self.status != "new": return # first, validate the transform if not self.validate(): raise ApplicationConfigurationError( None, "Validate failed for Transform %s" % self.name) self.updateStatus("running")
def loadPlugins(c): import sys from Ganga.Utility.logging import getLogger try: import Lib.Panda import Lib.Jedi except SystemExit: from Ganga.Core.exceptions import ApplicationConfigurationError import commands (s, o) = commands.getstatusoutput('curl --version') if (s): raise ApplicationConfigurationError( "Couldn't load Panda Client: ensure 'curl' is available") else: raise ApplicationConfigurationError("Couldn't load Panda Client") import Lib.Athena import Lib.Executable import Lib.ProdTrans return None
def getDBDatasets(jobO, trf, dbrelease): from pandatools import Client # get DB datasets dbrFiles = {} dbrDsList = [] if trf or dbrelease != '': if trf: # parse jobO for TRF tmpItems = jobO.split() else: # mimic a trf parameter to reuse following algorithm tmpItems = ['%DB=' + dbrelease] # look for DBRelease for tmpItem in tmpItems: match = re.search('%DB=([^:]+):(.+)$', tmpItem) if match: tmpDbrDS = match.group(1) tmpDbrLFN = match.group(2) # get files in the dataset if not tmpDbrDS in dbrDsList: logger.info("Querying files in dataset:%s" % tmpDbrDS) try: tmpList = Client.queryFilesInDataset(tmpDbrDS, False) except: raise ApplicationConfigurationError( None, "ERROR : error while looking up dataset %s. Perhaps this dataset does not exist?" % tmpDbrDS) # append for tmpLFN, tmpVal in tmpList.iteritems(): dbrFiles[tmpLFN] = tmpVal dbrDsList.append(tmpDbrDS) # check if tmpDbrLFN not in dbrFiles: raise ApplicationConfigurationError( None, "ERROR : %s is not in %s" % (tmpDbrLFN, tmpDbrDS)) return dbrFiles, dbrDsList
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if type(app.exe) == File: input_dir = app.getJobObject().getInputWorkspace().getPath() exefile = os.path.join(input_dir, os.path.basename(app.exe.name)) if not os.path.exists(exefile): msg = 'Executable: "%s" must exist!' % str(exefile) raise ApplicationConfigurationError(None, msg) os.system('chmod +x %s' % exefile) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))