def master_prepare(self,app,appconfig): '''Prepare the master job''' from pandatools import Client from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec job = app._getParent() logger.debug('ExecutablePandaRTHandler master_prepare called for %s', job.getFQID('.')) # set chirp variables if configPanda['chirpconfig'] or configPanda['chirpserver']: setChirpVariables() # Pack inputsandbox inputsandbox = 'sources.%s.tar' % commands.getoutput('uuidgen 2> /dev/null') inpw = job.getInputWorkspace() # add user script to inputsandbox if hasattr(job.application.exe, "name"): if not job.application.exe in job.inputsandbox: job.inputsandbox.append(job.application.exe) for fname in [f.name for f in job.inputsandbox]: fname.rstrip(os.sep) path = fname[:fname.rfind(os.sep)] f = fname[fname.rfind(os.sep)+1:] rc, output = commands.getstatusoutput('tar rf %s -C %s %s' % (inpw.getPath(inputsandbox), path, f)) if rc: logger.error('Packing inputsandbox failed with status %d',rc) logger.error(output) raise ApplicationConfigurationError('Packing inputsandbox failed.') if len(job.inputsandbox) > 0: rc, output = commands.getstatusoutput('gzip %s' % (inpw.getPath(inputsandbox))) if rc: logger.error('Packing inputsandbox failed with status %d',rc) logger.error(output) raise ApplicationConfigurationError('Packing inputsandbox failed.') inputsandbox += ".gz" else: inputsandbox = None # Upload Inputsandbox if inputsandbox: logger.debug('Uploading source tarball ...') uploadSources(inpw.getPath(),os.path.basename(inputsandbox)) self.inputsandbox = inputsandbox else: self.inputsandbox = None # input dataset if job.inputdata: if job.inputdata._name != 'DQ2Dataset': raise ApplicationConfigurationError('PANDA application supports only DQ2Datasets') # run brokerage here if not splitting if not job.splitter: from GangaPanda.Lib.Panda.Panda import runPandaBrokerage runPandaBrokerage(job) elif job.splitter._name not in ['DQ2JobSplitter', 'ArgSplitter', 'ArgSplitterTask']: raise ApplicationConfigurationError('Panda splitter must be DQ2JobSplitter or ArgSplitter') if job.backend.site == 'AUTO': raise ApplicationConfigurationError('site is still AUTO after brokerage!') # output dataset if job.outputdata: if job.outputdata._name != 'DQ2OutputDataset': raise ApplicationConfigurationError('Panda backend supports only DQ2OutputDataset') else: logger.info('Adding missing DQ2OutputDataset') job.outputdata = DQ2OutputDataset() job.outputdata.datasetname,outlfn = dq2outputdatasetname(job.outputdata.datasetname, job.id, job.outputdata.isGroupDS, job.outputdata.groupname) self.outDsLocation = Client.PandaSites[job.backend.site]['ddm'] try: Client.addDataset(job.outputdata.datasetname,False,location=self.outDsLocation) logger.info('Output dataset %s registered at %s'%(job.outputdata.datasetname,self.outDsLocation)) dq2_set_dataset_lifetime(job.outputdata.datasetname, location=self.outDsLocation) except exceptions.SystemExit: raise BackendError('Panda','Exception in Client.addDataset %s: %s %s'%(job.outputdata.datasetname,sys.exc_info()[0],sys.exc_info()[1])) # handle the libds if job.backend.libds: self.libDataset = job.backend.libds self.fileBO = getLibFileSpecFromLibDS(self.libDataset) self.library = self.fileBO.lfn elif job.backend.bexec: self.libDataset = job.outputdata.datasetname+'.lib' self.library = '%s.tgz' % self.libDataset try: Client.addDataset(self.libDataset,False,location=self.outDsLocation) dq2_set_dataset_lifetime(self.libDataset, location=self.outDsLocation) logger.info('Lib dataset %s registered at %s'%(self.libDataset,self.outDsLocation)) except exceptions.SystemExit: raise BackendError('Panda','Exception in Client.addDataset %s: %s %s'%(self.libDataset,sys.exc_info()[0],sys.exc_info()[1])) # collect extOutFiles self.extOutFile = [] for tmpName in job.outputdata.outputdata: if tmpName != '': self.extOutFile.append(tmpName) for tmpName in job.outputsandbox: if tmpName != '': self.extOutFile.append(tmpName) for tmpName in job.backend.extOutFile: if tmpName != '': self.extOutFile.append(tmpName) # create build job if job.backend.bexec != '': jspec = JobSpec() jspec.jobDefinitionID = job.id jspec.jobName = commands.getoutput('uuidgen 2> /dev/null') jspec.transformation = '%s/buildGen-00-00-01' % Client.baseURLSUB if Client.isDQ2free(job.backend.site): jspec.destinationDBlock = '%s/%s' % (job.outputdata.datasetname,self.libDataset) jspec.destinationSE = 'local' else: jspec.destinationDBlock = self.libDataset jspec.destinationSE = job.backend.site jspec.prodSourceLabel = configPanda['prodSourceLabelBuild'] jspec.processingType = configPanda['processingType'] jspec.assignedPriority = configPanda['assignedPriorityBuild'] jspec.computingSite = job.backend.site jspec.cloud = job.backend.requirements.cloud jspec.jobParameters = '-o %s' % (self.library) if self.inputsandbox: jspec.jobParameters += ' -i %s' % (self.inputsandbox) else: raise ApplicationConfigurationError('Executable on Panda with build job defined, but inputsandbox is emtpy !') matchURL = re.search('(http.*://[^/]+)/',Client.baseURLCSRVSSL) if matchURL: jspec.jobParameters += ' --sourceURL %s ' % matchURL.group(1) if job.backend.bexec != '': jspec.jobParameters += ' --bexec "%s" ' % urllib.quote(job.backend.bexec) jspec.jobParameters += ' -r %s ' % '.' fout = FileSpec() fout.lfn = self.library fout.type = 'output' fout.dataset = self.libDataset fout.destinationDBlock = self.libDataset jspec.addFile(fout) flog = FileSpec() flog.lfn = '%s.log.tgz' % self.libDataset flog.type = 'log' flog.dataset = self.libDataset flog.destinationDBlock = self.libDataset jspec.addFile(flog) return jspec else: return None
def master_prepare(self,app,appmasterconfig): # PandaTools from pandatools import Client from pandatools import AthenaUtils from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec job = app._getParent() logger.debug('AthenaMCPandaRTHandler master_prepare called for %s', job.getFQID('.')) usertag = configDQ2['usertag'] #usertag='user09' nickname = getNickname(allowMissingNickname=True) self.libDataset = '%s.%s.ganga.%s_%d.lib._%06d' % (usertag,nickname,commands.getoutput('hostname').split('.')[0],int(time.time()),job.id) # self.userprefix='%s.%s.ganga' % (usertag,gridProxy.identity()) sources = 'sources.%s.tar.gz' % commands.getoutput('uuidgen 2> /dev/null') self.library = '%s.lib.tgz' % self.libDataset # check DBRelease # if job.backend.dbRelease != '' and job.backend.dbRelease.find(':') == -1: # raise ApplicationConfigurationError(None,"ERROR : invalid argument for backend.dbRelease. Must be 'DatasetName:FileName'") # unpack library logger.debug('Creating source tarball ...') tmpdir = '/tmp/%s' % commands.getoutput('uuidgen 2> /dev/null') os.mkdir(tmpdir) inputbox=[] if os.path.exists(app.transform_archive): # must add a condition on size. inputbox += [ File(app.transform_archive) ] if app.evgen_job_option: self.evgen_job_option=app.evgen_job_option if os.path.exists(app.evgen_job_option): # locally modified job option file to add to the input sand box inputbox += [ File(app.evgen_job_option) ] self.evgen_job_option=app.evgen_job_option.split("/")[-1] # add input sandbox files if (job.inputsandbox): for file in job.inputsandbox: inputbox += [ file ] # add option files for extFile in job.backend.extOutFile: try: shutil.copy(extFile,tmpdir) except IOError: os.makedirs(tmpdir) shutil.copy(extFile,tmpdir) # fill the archive for opt_file in inputbox: try: shutil.copy(opt_file.name,tmpdir) except IOError: os.makedirs(tmpdir) shutil.copy(opt_file.name,tmpdir) # now tar it up again inpw = job.getInputWorkspace() rc, output = commands.getstatusoutput('tar czf %s -C %s .' % (inpw.getPath(sources),tmpdir)) if rc: logger.error('Packing sources failed with status %d',rc) logger.error(output) raise ApplicationConfigurationError(None,'Packing sources failed.') shutil.rmtree(tmpdir) # upload sources logger.debug('Uploading source tarball ...') try: cwd = os.getcwd() os.chdir(inpw.getPath()) rc, output = Client.putFile(sources) if output != 'True': logger.error('Uploading sources %s failed. Status = %d', sources, rc) logger.error(output) raise ApplicationConfigurationError(None,'Uploading archive failed') finally: os.chdir(cwd) # Use Panda's brokerage ## if job.inputdata and len(app.sites)>0: ## # update cloud, use inputdata's ## from dq2.info.TiersOfATLAS import whichCloud,ToACache ## inclouds=[] ## for site in app.sites: ## cloudSite=whichCloud(app.sites[0]) ## if cloudSite not in inclouds: ## inclouds.append(cloudSite) ## # now converting inclouds content into proper brokering stuff. ## outclouds=[] ## for cloudSite in inclouds: ## for cloudID, eachCloud in ToACache.dbcloud.iteritems(): ## if cloudSite==eachCloud: ## cloud=cloudID ## outclouds.append(cloud) ## break ## print outclouds ## # finally, matching with user's wishes ## if len(outclouds)>0: ## if not job.backend.requirements.cloud: # no user wish, update ## job.backend.requirements.cloud=outclouds[0] ## else: ## try: ## assert job.backend.requirements.cloud in outclouds ## except: ## raise ApplicationConfigurationError(None,'Input dataset not available in target cloud %s. Please try any of the following %s' % (job.backend.requirements.cloud, str(outclouds))) from GangaPanda.Lib.Panda.Panda import runPandaBrokerage runPandaBrokerage(job) if job.backend.site == 'AUTO': raise ApplicationConfigurationError(None,'site is still AUTO after brokerage!') # output dataset preparation and registration try: outDsLocation = Client.PandaSites[job.backend.site]['ddm'] except: raise ApplicationConfigurationError(None,"Could not extract output dataset location from job.backend.site value: %s. Aborting" % job.backend.site) if not app.dryrun: for outtype in app.outputpaths.keys(): dset=string.replace(app.outputpaths[outtype],"/",".") dset=dset[1:] # dataset registration must be done only once. print "registering output dataset %s at %s" % (dset,outDsLocation) try: Client.addDataset(dset,False,location=outDsLocation) dq2_set_dataset_lifetime(dset, location=outDsLocation) except: raise ApplicationConfigurationError(None,"Fail to create output dataset %s. Aborting" % dset) # extend registration to build job lib dataset: print "registering output dataset %s at %s" % (self.libDataset,outDsLocation) try: Client.addDataset(self.libDataset,False,location=outDsLocation) dq2_set_dataset_lifetime(self.libDataset, outDsLocation) except: raise ApplicationConfigurationError(None,"Fail to create output dataset %s. Aborting" % self.libDataset) ### cacheVer = "-AtlasProduction_" + str(app.prod_release) logger.debug("master job submit?") self.outsite=job.backend.site if app.se_name and app.se_name != "none" and not self.outsite: self.outsite=app.se_name # create build job jspec = JobSpec() jspec.jobDefinitionID = job.id jspec.jobName = commands.getoutput('uuidgen 2> /dev/null') jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel jspec.homepackage = 'AnalysisTransforms'+cacheVer#+nightVer jspec.transformation = '%s/buildJob-00-00-03' % Client.baseURLSUB # common base to Athena and AthenaMC jobs: buildJob is a pilot job which takes care of all inputs for the real jobs (in prepare() jspec.destinationDBlock = self.libDataset jspec.destinationSE = job.backend.site jspec.prodSourceLabel = 'panda' jspec.assignedPriority = 2000 jspec.computingSite = job.backend.site jspec.cloud = job.backend.requirements.cloud # jspec.jobParameters = self.args not known yet jspec.jobParameters = '-o %s' % (self.library) if app.userarea: print app.userarea jspec.jobParameters += ' -i %s' % (os.path.basename(app.userarea)) else: jspec.jobParameters += ' -i %s' % (sources) jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel) matchURL = re.search('(http.*://[^/]+)/',Client.baseURLSSL) if matchURL: jspec.jobParameters += ' --sourceURL %s' % matchURL.group(1) fout = FileSpec() fout.lfn = self.library fout.type = 'output' fout.dataset = self.libDataset fout.destinationDBlock = self.libDataset jspec.addFile(fout) flog = FileSpec() flog.lfn = '%s.log.tgz' % self.libDataset flog.type = 'log' flog.dataset = self.libDataset flog.destinationDBlock = self.libDataset jspec.addFile(flog) #print "MASTER JOB DETAILS:",jspec.jobParameters return jspec
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): """Prepare the specific aspec of each subjob. Returns: subjobconfig list of objects understood by backends.""" from pandatools import Client from pandatools import AthenaUtils from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_set_dataset_lifetime from GangaPanda.Lib.Panda.Panda import refreshPandaSpecs # make sure we have the correct siteType refreshPandaSpecs() job = app._getParent() masterjob = job._getRoot() logger.debug('ProdTransPandaRTHandler prepare called for %s', job.getFQID('.')) job.backend.actualCE = job.backend.site job.backend.requirements.cloud = Client.PandaSites[job.backend.site]['cloud'] # check that the site is in a submit-able status if not job.splitter or job.splitter._name != 'DQ2JobSplitter': allowed_sites = job.backend.list_ddm_sites() try: outDsLocation = Client.PandaSites[job.backend.site]['ddm'] tmpDsExist = False if (configPanda['processingType'].startswith('gangarobot') or configPanda['processingType'].startswith('hammercloud')): #if Client.getDatasets(job.outputdata.datasetname): if getDatasets(job.outputdata.datasetname): tmpDsExist = True logger.info('Re-using output dataset %s'%job.outputdata.datasetname) if not configPanda['specialHandling']=='ddm:rucio' and not configPanda['processingType'].startswith('gangarobot') and not configPanda['processingType'].startswith('hammercloud') and not configPanda['processingType'].startswith('rucio_test'): Client.addDataset(job.outputdata.datasetname,False,location=outDsLocation,allowProdDisk=True,dsExist=tmpDsExist) logger.info('Output dataset %s registered at %s'%(job.outputdata.datasetname,outDsLocation)) dq2_set_dataset_lifetime(job.outputdata.datasetname, outDsLocation) except exceptions.SystemExit: raise BackendError('Panda','Exception in adding dataset %s: %s %s'%(job.outputdata.datasetname,sys.exc_info()[0],sys.exc_info()[1])) # JobSpec. jspec = JobSpec() jspec.currentPriority = app.priority jspec.jobDefinitionID = masterjob.id jspec.jobName = commands.getoutput('uuidgen 2> /dev/null') jspec.coreCount = app.core_count jspec.AtlasRelease = 'Atlas-%s' % app.atlas_release jspec.homepackage = app.home_package jspec.transformation = app.transformation jspec.destinationDBlock = job.outputdata.datasetname if job.outputdata.location: jspec.destinationSE = job.outputdata.location else: jspec.destinationSE = job.backend.site if job.inputdata: jspec.prodDBlock = job.inputdata.dataset[0] else: jspec.prodDBlock = 'NULL' if app.prod_source_label: jspec.prodSourceLabel = app.prod_source_label else: jspec.prodSourceLabel = configPanda['prodSourceLabelRun'] jspec.processingType = configPanda['processingType'] jspec.specialHandling = configPanda['specialHandling'] jspec.computingSite = job.backend.site jspec.cloud = job.backend.requirements.cloud jspec.cmtConfig = app.atlas_cmtconfig if app.dbrelease == 'LATEST': try: latest_dbrelease = getLatestDBReleaseCaching() except: from pandatools import Client latest_dbrelease = Client.getLatestDBRelease() m = re.search('(.*):DBRelease-(.*)\.tar\.gz', latest_dbrelease) if m: self.dbrelease_dataset = m.group(1) self.dbrelease = m.group(2) else: raise ApplicationConfigurationError(None, "Error retrieving LATEST DBRelease. Try setting application.dbrelease manually.") else: self.dbrelease_dataset = app.dbrelease_dataset self.dbrelease = app.dbrelease jspec.jobParameters = app.job_parameters if self.dbrelease: if self.dbrelease == 'current': jspec.jobParameters += ' --DBRelease=current' else: if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --DBRelease=DBRelease-%s.tar.gz' % (self.dbrelease,) else: jspec.jobParameters += ' DBRelease=DBRelease-%s.tar.gz' % (self.dbrelease,) dbspec = FileSpec() dbspec.lfn = 'DBRelease-%s.tar.gz' % self.dbrelease dbspec.dataset = self.dbrelease_dataset dbspec.prodDBlock = jspec.prodDBlock dbspec.type = 'input' jspec.addFile(dbspec) if job.inputdata: m = re.search('(.*)\.(.*)\.(.*)\.(.*)\.(.*)\.(.*)', job.inputdata.dataset[0]) if not m: logger.error("Error retrieving run number from dataset name") #raise ApplicationConfigurationError(None, "Error retrieving run number from dataset name") runnumber = 105200 else: runnumber = int(m.group(2)) if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --runNumber %d' % runnumber else: jspec.jobParameters += ' RunNumber=%d' % runnumber # Output files. randomized_lfns = [] ilfn = 0 for lfn, lfntype in zip(app.output_files,app.output_type): ofspec = FileSpec() if app.randomize_lfns: randomized_lfn = lfn + ('.%s.%d.%s' % (job.backend.site, int(time.time()), commands.getoutput('uuidgen 2> /dev/null')[:4] ) ) else: randomized_lfn = lfn ofspec.lfn = randomized_lfn randomized_lfns.append(randomized_lfn) ofspec.destinationDBlock = jspec.destinationDBlock ofspec.destinationSE = jspec.destinationSE ofspec.dataset = jspec.destinationDBlock ofspec.type = 'output' jspec.addFile(ofspec) if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --output%sFile %s' % (lfntype, randomized_lfns[ilfn]) else: jspec.jobParameters += ' output%sFile=%s' % (lfntype, randomized_lfns[ilfn]) ilfn=ilfn+1 # Input files. if job.inputdata: for guid, lfn, size, checksum, scope in zip(job.inputdata.guids, job.inputdata.names, job.inputdata.sizes, job.inputdata.checksums, job.inputdata.scopes): ifspec = FileSpec() ifspec.lfn = lfn ifspec.GUID = guid ifspec.fsize = size ifspec.md5sum = checksum ifspec.scope = scope ifspec.dataset = jspec.prodDBlock ifspec.prodDBlock = jspec.prodDBlock ifspec.type = 'input' jspec.addFile(ifspec) if app.input_type: itype = app.input_type else: itype = m.group(5) if jspec.transformation.endswith("_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join(job.inputdata.names)) else: jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join(job.inputdata.names)) # Log files. lfspec = FileSpec() lfspec.lfn = '%s.job.log.tgz' % jspec.jobName lfspec.destinationDBlock = jspec.destinationDBlock lfspec.destinationSE = jspec.destinationSE lfspec.dataset = jspec.destinationDBlock lfspec.type = 'log' jspec.addFile(lfspec) return jspec
def master_prepare(self, app, appconfig): '''Prepare the master job''' from pandatools import Client from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec job = app._getParent() logger.debug('ExecutablePandaRTHandler master_prepare called for %s', job.getFQID('.')) # set chirp variables if configPanda['chirpconfig'] or configPanda['chirpserver']: setChirpVariables() # Pack inputsandbox inputsandbox = 'sources.%s.tar' % commands.getoutput( 'uuidgen 2> /dev/null') inpw = job.getInputWorkspace() # add user script to inputsandbox if hasattr(job.application.exe, "name"): if not job.application.exe in job.inputsandbox: job.inputsandbox.append(job.application.exe) for fname in [f.name for f in job.inputsandbox]: fname.rstrip(os.sep) path = fname[:fname.rfind(os.sep)] f = fname[fname.rfind(os.sep) + 1:] rc, output = commands.getstatusoutput( 'tar rf %s -C %s %s' % (inpw.getPath(inputsandbox), path, f)) if rc: logger.error('Packing inputsandbox failed with status %d', rc) logger.error(output) raise ApplicationConfigurationError( None, 'Packing inputsandbox failed.') if len(job.inputsandbox) > 0: rc, output = commands.getstatusoutput('gzip %s' % (inpw.getPath(inputsandbox))) if rc: logger.error('Packing inputsandbox failed with status %d', rc) logger.error(output) raise ApplicationConfigurationError( None, 'Packing inputsandbox failed.') inputsandbox += ".gz" else: inputsandbox = None # Upload Inputsandbox if inputsandbox: logger.debug('Uploading source tarball ...') uploadSources(inpw.getPath(), os.path.basename(inputsandbox)) self.inputsandbox = inputsandbox else: self.inputsandbox = None # input dataset if job.inputdata: if job.inputdata._name != 'DQ2Dataset': raise ApplicationConfigurationError( None, 'PANDA application supports only DQ2Datasets') # run brokerage here if not splitting if not job.splitter: from GangaPanda.Lib.Panda.Panda import runPandaBrokerage runPandaBrokerage(job) elif job.splitter._name not in [ 'DQ2JobSplitter', 'ArgSplitter', 'ArgSplitterTask' ]: raise ApplicationConfigurationError( None, 'Panda splitter must be DQ2JobSplitter or ArgSplitter') if job.backend.site == 'AUTO': raise ApplicationConfigurationError( None, 'site is still AUTO after brokerage!') # output dataset if job.outputdata: if job.outputdata._name != 'DQ2OutputDataset': raise ApplicationConfigurationError( None, 'Panda backend supports only DQ2OutputDataset') else: logger.info('Adding missing DQ2OutputDataset') job.outputdata = DQ2OutputDataset() job.outputdata.datasetname, outlfn = dq2outputdatasetname( job.outputdata.datasetname, job.id, job.outputdata.isGroupDS, job.outputdata.groupname) self.outDsLocation = Client.PandaSites[job.backend.site]['ddm'] try: Client.addDataset(job.outputdata.datasetname, False, location=self.outDsLocation) logger.info('Output dataset %s registered at %s' % (job.outputdata.datasetname, self.outDsLocation)) dq2_set_dataset_lifetime(job.outputdata.datasetname, location=self.outDsLocation) except exceptions.SystemExit: raise BackendError( 'Panda', 'Exception in Client.addDataset %s: %s %s' % (job.outputdata.datasetname, sys.exc_info()[0], sys.exc_info()[1])) # handle the libds if job.backend.libds: self.libDataset = job.backend.libds self.fileBO = getLibFileSpecFromLibDS(self.libDataset) self.library = self.fileBO.lfn elif job.backend.bexec: self.libDataset = job.outputdata.datasetname + '.lib' self.library = '%s.tgz' % self.libDataset try: Client.addDataset(self.libDataset, False, location=self.outDsLocation) dq2_set_dataset_lifetime(self.libDataset, location=self.outDsLocation) logger.info('Lib dataset %s registered at %s' % (self.libDataset, self.outDsLocation)) except exceptions.SystemExit: raise BackendError( 'Panda', 'Exception in Client.addDataset %s: %s %s' % (self.libDataset, sys.exc_info()[0], sys.exc_info()[1])) # collect extOutFiles self.extOutFile = [] for tmpName in job.outputdata.outputdata: if tmpName != '': self.extOutFile.append(tmpName) for tmpName in job.outputsandbox: if tmpName != '': self.extOutFile.append(tmpName) for tmpName in job.backend.extOutFile: if tmpName != '': self.extOutFile.append(tmpName) # create build job if job.backend.bexec != '': jspec = JobSpec() jspec.jobDefinitionID = job.id jspec.jobName = commands.getoutput('uuidgen 2> /dev/null') jspec.transformation = '%s/buildGen-00-00-01' % Client.baseURLSUB if Client.isDQ2free(job.backend.site): jspec.destinationDBlock = '%s/%s' % ( job.outputdata.datasetname, self.libDataset) jspec.destinationSE = 'local' else: jspec.destinationDBlock = self.libDataset jspec.destinationSE = job.backend.site jspec.prodSourceLabel = configPanda['prodSourceLabelBuild'] jspec.processingType = configPanda['processingType'] jspec.assignedPriority = configPanda['assignedPriorityBuild'] jspec.computingSite = job.backend.site jspec.cloud = job.backend.requirements.cloud jspec.jobParameters = '-o %s' % (self.library) if self.inputsandbox: jspec.jobParameters += ' -i %s' % (self.inputsandbox) else: raise ApplicationConfigurationError( None, 'Executable on Panda with build job defined, but inputsandbox is emtpy !' ) matchURL = re.search('(http.*://[^/]+)/', Client.baseURLCSRVSSL) if matchURL: jspec.jobParameters += ' --sourceURL %s ' % matchURL.group(1) if job.backend.bexec != '': jspec.jobParameters += ' --bexec "%s" ' % urllib.quote( job.backend.bexec) jspec.jobParameters += ' -r %s ' % '.' fout = FileSpec() fout.lfn = self.library fout.type = 'output' fout.dataset = self.libDataset fout.destinationDBlock = self.libDataset jspec.addFile(fout) flog = FileSpec() flog.lfn = '%s.log.tgz' % self.libDataset flog.type = 'log' flog.dataset = self.libDataset flog.destinationDBlock = self.libDataset jspec.addFile(flog) return jspec else: return None
def retry(self,JobsetID,newSite=False,newOpts={},noSubmit=False,ignoreDuplication=False,useJobsetID=False,retryBuild=False,reproduceFiles=[],unsetRetryID=False): # get logger tmpLog = PLogger.getPandaLogger() # check proxy self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose, useCache=True) # force update just in case self.status(JobsetID,True) # set an empty map since mutable default value is used if newOpts == {}: newOpts = {} # get jobset newJobsetID = -1 jobList = self.getJobIDsWithSetID(JobsetID) if jobList == None: # works only for jobsetID if useJobsetID: return # works with jobID isJobset = False jobList = [JobsetID] else: isJobset = True tmpMsg = "ID=%s is composed of JobID=" % JobsetID for tmpJobID in jobList: tmpMsg += '%s,' % tmpJobID tmpMsg = tmpMsg[:-1] tmpLog.info(tmpMsg) for JobID in jobList: # get job info from local repository localJob = self.getJobInfo(JobID) if localJob == None: tmpLog.warning("JobID=%s not found in local repository. Synchronization may be needed" % JobID) return None # for JEDI if localJob.isJEDI(): status,out = Client.retryTask( localJob.jediTaskID, verbose=self.verbose, properErrorCode=True, newParams=newOpts) if status != 0: tmpLog.error(status) tmpLog.error(out) tmpLog.error("Failed to retry TaskID=%s" % localJob.jediTaskID) return False tmpStat,tmpDiag = out if (not tmpStat in [0,True] and newOpts == {}) or (newOpts != {} and tmpStat != 3): tmpLog.error(tmpDiag) tmpLog.error("Failed to retry TaskID=%s" % localJob.jediTaskID) return False tmpLog.info(tmpDiag) continue # skip running job if localJob.dbStatus != 'frozen': tmpLog.info('Retry failed subjobs in running jobId=%s' % JobID) status,out = Client.retryFailedJobsInActive(JobID,verbose=self.verbose) if status != 0: tmpLog.error(status) tmpLog.error(out) tmpLog.error("Failed to retry JobID=%s" % JobID) else: job = self.status(JobID) if isJobset: continue else: return # skip already retried if localJob.retryID != '0': if isJobset: tmpLog.info('Skip JobID=%s since already retried by JobID=%s JobsetID=%s' % \ (JobID,localJob.retryID,localJob.retryJobsetID)) continue else: tmpLog.warning('This job was already retried by JobID=%s' % localJob.retryID) return # check status of buildJob if not retryBuild and not localJob.buildStatus in ['','finished']: tmpMsgStr = 'Cannot retry since status of buildJob %s is %s (!= finished). ' \ % (localJob.PandaID.split(',')[0],localJob.buildStatus) tmpMsgStr += 'Please execute %s with the same input/output datasets (or containers). ' % localJob.jobType tmpMsgStr += 'It will run only on failed/cancelled/unused input files ' tmpMsgStr += 'and append output files to the output dataset container. ' tmpMsgStr += 'Or you may set retryBuild=True in pbook.retry() ' tmpLog.warning(tmpMsgStr) if isJobset: continue else: return # check opts for newSite if newSite or newOpts != {}: if not localJob.outDS.endswith('/') and not newOpts.has_key('outDS') and not newOpts.has_key('--outDS'): tmpLog.warning('You need to specify --outDS in newOpts to retry at new site unless container is used as output') return # get list of failed jobs pandaIDs = localJob.PandaID.split(',') statusList= localJob.jobStatus.split(',') jobList = [] for idx in range(len(pandaIDs)): # check status unless reproduce files if reproduceFiles == [] and not statusList[idx] in ['failed','cancelled']: continue jobList.append(pandaIDs[idx]) # no failed job if jobList == []: if isJobset: tmpLog.info('Skip JobID=%s since no failed jobs' % JobID) continue else: tmpLog.info('No failed jobs to be retried for JobID=%s' % JobID) return # get full job spec tmpLog.info("Retrying JobID=%s ..." % JobID) tmpLog.info("Getting job info") idxJL = 0 nQuery = 500 pandaJobs = [] while idxJL < len(jobList): # avoid burst query tmpLog.info(" %5s/%s" % (idxJL,len(jobList))) status,oTmp = Client.getFullJobStatus( jobList[idxJL:idxJL+nQuery], verbose=self.verbose) if status != 0: tmpLog.error(status) tmpLog.error(oTmp) tmpLog.error("Cannot get job info from Panda server") return pandaJobs += oTmp idxJL += nQuery time.sleep(1) tmpLog.info(" %5s/%s" % (len(jobList),len(jobList))) # get PandaIDs to reproduce files if reproduceFiles != []: # change wildcard to .* for regexp reproduceFilePatt = [] for tmpReproduceFile in reproduceFiles: if '*' in tmpReproduceFile: tmpReproduceFile = tmpReproduceFile.replace('*','.*') reproduceFilePatt.append(tmpReproduceFile) # get list of jobs which produced interesting files tmpJobList = [] tmpPandaJobs = [] for tmpPandaJob in pandaJobs: # check names tmpMatchFlag = False for tmpFile in tmpPandaJob.Files: if tmpFile.type == 'output' and tmpFile.status == 'ready': for tmpReproduceFile in reproduceFilePatt: # normal matching if tmpReproduceFile == tmpFile.lfn: tmpMatchFlag = True break # wild card if '*' in tmpReproduceFile and \ re.search('^'+tmpReproduceFile,tmpFile.lfn) != None: tmpMatchFlag = True break if tmpMatchFlag: break # append if tmpMatchFlag: tmpJobList.append(tmpPandaJob.PandaID) tmpPandaJobs.append(tmpPandaJob) # use new list jobList = tmpJobList pandaJobs = tmpPandaJobs if jobList == []: tmpLog.info("No jobs to reproduce files : Jobs in JobID=%s didn't produce lost files" % JobID) continue # jobdefID newJobdefID = PsubUtils.readJobDefID() # reset some parameters retryJobs = [] retrySite = None retryElement = None retryDestSE = None outDsName = None shadowList = [] oldLibDS = None newLibDS = None newLibTgz = None rebroMap = {} for idx in range(len(jobList)): job = pandaJobs[idx] # skip exired if job == None: tmpLog.warning("Could not retry jobs older than 30 days : JobID=%s (PandaID=%s) expired" \ % (JobID,jobList[idxJob])) return # skip jobs reassigned by rebrokerage if (job.jobStatus == 'cancelled' and job.taskBufferErrorCode in [105,'105']) or \ (job.jobStatus == 'failed' and job.taskBufferErrorCode in [106,'106']): # extract JobIDs of reassigned jobs tmpM = re.search('JobsetID=(\d+) JobID=(\d+)',job.taskBufferErrorDiag) if tmpM != None: tmpRebKey = (tmpM.group(1),tmpM.group(2)) if not rebroMap.has_key(tmpRebKey): rebroMap[tmpRebKey] = 0 # count # of reassigned jobs rebroMap[tmpRebKey] += 1 continue # get shadow list if (not ignoreDuplication) and outDsName == None and job.prodSourceLabel == 'user': # look for dataset for log since it doesn't have suffix even when --individualOutDS is used for tmpFile in job.Files: if tmpFile.type == 'log': outDsName = tmpFile.dataset break # output dataset was not found if outDsName == None: tmpLog.error("Could not get output dataset name for JobID=%s (PandaID=%s)" \ % (JobID,job.PandaID)) return # get files in shadow if outDsName.endswith('/'): shadowList = Client.getFilesInShadowDataset( outDsName, Client.suffixShadow, self.verbose) else: # disable duplication check mainly for old overlay jobs since non-signal files are wrongly skipped #shadowList = Client.getFilesInShadowDatasetOld(outDsName,Client.suffixShadow,self.verbose) pass # unify sitename if retrySite == None: retrySite = job.computingSite retryElement = job.computingElement retryDestSE = job.destinationSE # reset job.jobStatus = None job.commandToPilot = None job.startTime = None job.endTime = None job.attemptNr = 1+job.attemptNr for attr in job._attributes: if attr.endswith('ErrorCode') or attr.endswith('ErrorDiag'): setattr(job,attr,None) job.transExitCode = None job.computingSite = retrySite job.computingElement = retryElement job.destinationSE = retryDestSE job.dispatchDBlock = None if not unsetRetryID: job.jobExecutionID = JobID job.jobDefinitionID = newJobdefID job.parentID = job.PandaID if job.jobsetID != ['NULL',None,-1]: if not unsetRetryID: job.sourceSite = job.jobsetID job.jobsetID = newJobsetID skipInputList = [] numUsedFiles = 0 # loop over all files for file in job.Files: file.rowID = None if file.type == 'input': # protection against wrong sync which doesn't update buildStatus correctly if not retryBuild and file.lfn.endswith('.lib.tgz') and file.GUID == 'NULL': tmpLog.warning('GUID for %s is unknown. Cannot retry when corresponding buildJob failed' \ % file.lfn) return if not retryBuild or not file.lfn.endswith('.lib.tgz'): file.status = 'ready' # set new lib dataset if retryBuild and file.lfn.endswith('.lib.tgz'): if newLibTgz != None: file.lfn = newLibTgz file.dataset = newLibDS file.dispatchDBlock = newLibDS # check with shadow for non lib.tgz/DBR tmpDbrMatch = re.search('^DBRelease-.*\.tar\.gz$',file.lfn) if tmpDbrMatch == None and not file.lfn.endswith('.lib.tgz'): if file.lfn in shadowList: skipInputList.append(file) else: numUsedFiles += 1 elif file.type in ('output','log'): file.destinationSE = retryDestSE file.destinationDBlock = re.sub('_sub\d+$','',file.destinationDBlock) # add retry num if file.dataset.endswith('/') or job.prodSourceLabel == 'panda': oldOutDsName = file.destinationDBlock retryDsPatt = '_r' if reproduceFiles != []: retryDsPatt = '_rp' retryMatch = re.search(retryDsPatt+'(\d+)$',file.destinationDBlock) if retryMatch == None: file.destinationDBlock += (retryDsPatt+'1') else: tmpDestinationDBlock = re.sub(retryDsPatt+'(\d+)$','',file.destinationDBlock) file.destinationDBlock = tmpDestinationDBlock + retryDsPatt + '%d' % (1+int(retryMatch.group(1))) if job.processingType == 'usermerge': job.jobParameters = job.jobParameters.replace(' %s ' % oldOutDsName, ' %s ' % file.destinationDBlock) # use new dataset name for buildXYZ if job.prodSourceLabel == 'panda': if file.lfn.endswith('.lib.tgz'): # get new libDS and lib.tgz names oldLibDS = file.dataset file.dataset = file.destinationDBlock newLibDS = file.dataset file.lfn = re.sub(oldLibDS,newLibDS,file.lfn) newLibTgz = file.lfn else: file.dataset = file.destinationDBlock # add attempt nr oldName = file.lfn if job.prodSourceLabel == 'panda' and file.lfn.endswith('.lib.tgz'): continue else: # append attempt number at the tail file.lfn = re.sub("\.\d+$","",file.lfn) file.lfn = "%s.%d" % (file.lfn,job.attemptNr) newName = file.lfn # modify jobParameters job.jobParameters = re.sub("'%s'" % oldName ,"'%s'" % newName, job.jobParameters) # look for output in trf oldGenelicName = re.sub('\.\d+$','',oldName) match = re.search(oldGenelicName+'(\.\d+)*(%20|")',job.jobParameters) if match != None: job.jobParameters = job.jobParameters.replace(match.group(0),newName+match.group(2)) # change lib.tgz name if retryBuild and newLibDS != None: job.jobParameters = re.sub(oldLibDS,newLibDS,job.jobParameters) # change destinationDBlock if job.prodSourceLabel == 'panda': job.destinationDBlock = newLibDS # all files are used by others if numUsedFiles == 0 and skipInputList != []: continue # remove skipped files strSkipped = '' for tmpFile in skipInputList: strSkipped += '%s,' % tmpFile.lfn job.Files.remove(tmpFile) strSkipped = strSkipped[:-1] # modify jobpar if strSkipped != '': optionToSkipFiles = '--skipInputByRetry' if not optionToSkipFiles in job.jobParameters: # just append job.jobParameters += "%s=%s " % (optionToSkipFiles,strSkipped) else: # extract already skipped files tmpMatch = re.search("(%s=[^ ]+)",job.jobParameters) if tmpMatch == None: tmpLog.error("Failed to extract arg of %s for PandaID=%s" \ % (optionToSkipFiles,job.PandaID)) return # replace job.jobParameters = re.sub(tmpMatch.group(1),"%s,%s" % (tmpMatch.group(1),optionToSkipFiles), job.jobParameters) if self.verbose: tmpLog.debug(job.jobParameters) # append retryJobs.append(job) # info on rebrokeage if rebroMap != {}: for tmpRebKey,tmpRebNumJobs in rebroMap.iteritems(): tmpRebSetID,tmpRebJobID = tmpRebKey tmpLog.info('Skip %s jobs since JobID=%s JobsetID=%s already reassigned them to another site' % \ (tmpRebNumJobs,tmpRebJobID,tmpRebSetID)) if retryJobs == []: tmpLog.info("No more jobs to be retried for JobID=%s" % JobID) if isJobset: continue else: return # all input files were or are being used by other jobs if retryJobs == []: tmpLog.info('All input files were or are being used by other jobs for the same output. No jobs to be retried. If you need to ignore duplication check (e.g., using the same EVNT file for multiple simulation subjobs), set ignoreDuplication=True. i.e. retry(123,ignoreDuplication=True)') if isJobset: continue else: return # check voms role if not retryJobs[0].workingGroup in ['NULL',None,'']: # VOMS role was used if not "--workingGroup" in job.metadata: # extract voms roles from metadata match = re.search("--voms( |=)[ \"]*([^ \"]+)",job.metadata) if match != None: vomsRoles = match.group(2) else: vomsRoles = "atlas:/atlas/%s/Role=production" % retryJobs[0].workingGroup # regenerate proxy with VOMS roles try: tmpLog.info("Checking proxy role to resubmit %s jobs" % retryJobs[0].workingGroup) self.gridPassPhrase,self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose,vomsRoles, useCache=True) except: tmpLog.error("Failed to generate a proxy with %s" % vomsRoles) return # check runtime env for new site submission if (newSite or newOpts != {}): if retryJobs[0].processingType == 'pathena' or '--useAthenaPackages' in retryJobs[0].metadata: from pandatools import AthenaUtils stA,retA = AthenaUtils.getAthenaVer() if not stA: tmpLog.error("Failed to get Athena rel/cache version in current runtime env") return athenaVer = retA['athenaVer'] cacheVer = retA['cacheVer'] nightVer = retA['nightVer'] wrongSetup = False if retryJobs[0].AtlasRelease != 'Atlas-%s' % athenaVer: wrongSetup = True errMsg = "Current Athena version Atlas-%s is inconsitent with the previous submission %s. " % (athenaVer,retryJobs[0].AtlasRelease) elif retryJobs[0].homepackage != 'AnalysisTransforms'+cacheVer+nightVer: wrongSetup = True errMsg = "Current cache version %s is inconsitent with the previous submission. " % cacheVer.replace('-','').replace('_','-') if wrongSetup: errMsg += 'You need to have the same runtime env as before since all job spec need to be re-created to send jobs to a new site. ' errMsg += 'Please setup Athena correctly and restart pbook' tmpLog.error(errMsg) return # test mode if noSubmit: continue # invoke pathena/prun to send job to new site if (newSite or newOpts != {}) and retryJobs[0].processingType != 'usermerge': # set parent jobID and jobsetID newOpts['provenanceID'] = retryJobs[0].jobExecutionID newOpts['panda_parentJobsetID'] = retryJobs[0].sourceSite tmpLog.info("Constructing job spec again to be sent to another site ...") comStat= PsubUtils.execWithModifiedParams(retryJobs,newOpts,self.verbose,newSite) if comStat == 0: # update database time.sleep(2) self.sync() else: tmpLog.error("Failed to submit jobs to Panda server") return # register datasets tmpOutDsLocation = Client.PandaSites[retryJobs[-1].computingSite]['ddm'] addedDataset = [] shadowDSname = None for tmpFile in retryJobs[-1].Files: if tmpFile.type in ['output','log'] and tmpFile.dataset.endswith('/'): # add shadow """ removed shadow if shadowDSname == None and tmpFile.type == 'log': shadowDSname = "%s%s" % (tmpFile.destinationDBlock,Client.suffixShadow) Client.addDataset(shadowDSname,self.verbose) """ # add datasets if not tmpFile.destinationDBlock in addedDataset: # create dataset Client.addDataset( tmpFile.destinationDBlock, self.verbose, location=tmpOutDsLocation, dsCheck=False) # add to container Client.addDatasetsToContainer( tmpFile.dataset, [tmpFile.destinationDBlock], self.verbose) # append addedDataset.append(tmpFile.destinationDBlock) # register libDS if retryBuild and newLibDS != None: Client.addDataset( newLibDS, self.verbose, location=tmpOutDsLocation, dsCheck=False) # submit tmpLog.info("Submitting job ...") status,out = Client.submitJobs(retryJobs,verbose=self.verbose) if out == None or status != 0: tmpLog.error(status) tmpLog.error(out) tmpLog.error("Failed to submit jobs to Panda server") return # update database pandaIDstatus = {} newJobID = None for items in out: # get newJobID if newJobID == None: newJobID = items[1] # check PandaID PandaID = items[0] if PandaID == 'NULL': tmpLog.error("Panda server returned wrong IDs. It may have a temporary problem") return # set newJobsetID if newJobsetID in [None,-1]: newJobsetID = items[2]['jobsetID'] # dummy statuso pandaIDstatus[PandaID] = ('defined','NULL') # set retry ID if not unsetRetryID: localJob.retryID = newJobID if not newJobsetID in [None,-1,'NULL']: localJob.retryJobsetID = newJobsetID try: PdbUtils.updateJobDB(localJob,self.verbose) except: tmpLog.error("Failed to set retryID for JobID=%s" % JobID) return # set new paramers newLocalJob = PdbUtils.convertPtoD(retryJobs,pandaIDstatus) newLocalJob.JobID = newJobID if not newJobsetID in [None,-1,'NULL']: newLocalJob.groupID = newJobsetID newLocalJob.creationTime = datetime.datetime.utcnow() # insert to DB try: PdbUtils.insertJobDB(newLocalJob,self.verbose) except: tmpLog.error("Failed to insert JobID=%s to local repository" % newJobID) return # write new jobdefID PsubUtils.writeJobDefID(newJobID) # done tmpMsg = 'Done. New JobID=%s' % newJobID if not newJobsetID in [None,-1,'NULL']: tmpMsg += " JobsetID=%s" % newJobsetID tmpLog.info(tmpMsg)
def prepare(self, app, appsubconfig, appmasterconfig, jobmasterconfig): """Prepare the specific aspec of each subjob. Returns: subjobconfig list of objects understood by backends.""" from pandatools import Client from pandatools import AthenaUtils from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec from GangaAtlas.Lib.ATLASDataset.DQ2Dataset import dq2_set_dataset_lifetime from GangaPanda.Lib.Panda.Panda import refreshPandaSpecs # make sure we have the correct siteType refreshPandaSpecs() job = app._getParent() masterjob = job._getRoot() logger.debug('ProdTransPandaRTHandler prepare called for %s', job.getFQID('.')) job.backend.actualCE = job.backend.site job.backend.requirements.cloud = Client.PandaSites[ job.backend.site]['cloud'] # check that the site is in a submit-able status if not job.splitter or job.splitter._name != 'DQ2JobSplitter': allowed_sites = job.backend.list_ddm_sites() try: outDsLocation = Client.PandaSites[job.backend.site]['ddm'] tmpDsExist = False if (configPanda['processingType'].startswith('gangarobot') or configPanda['processingType'].startswith('hammercloud')): #if Client.getDatasets(job.outputdata.datasetname): if getDatasets(job.outputdata.datasetname): tmpDsExist = True logger.info('Re-using output dataset %s' % job.outputdata.datasetname) if not configPanda[ 'specialHandling'] == 'ddm:rucio' and not configPanda[ 'processingType'].startswith( 'gangarobot' ) and not configPanda['processingType'].startswith( 'hammercloud') and not configPanda[ 'processingType'].startswith('rucio_test'): Client.addDataset(job.outputdata.datasetname, False, location=outDsLocation, allowProdDisk=True, dsExist=tmpDsExist) logger.info('Output dataset %s registered at %s' % (job.outputdata.datasetname, outDsLocation)) dq2_set_dataset_lifetime(job.outputdata.datasetname, outDsLocation) except exceptions.SystemExit: raise BackendError( 'Panda', 'Exception in adding dataset %s: %s %s' % (job.outputdata.datasetname, sys.exc_info()[0], sys.exc_info()[1])) # JobSpec. jspec = JobSpec() jspec.currentPriority = app.priority jspec.jobDefinitionID = masterjob.id jspec.jobName = commands.getoutput('uuidgen 2> /dev/null') jspec.coreCount = app.core_count jspec.AtlasRelease = 'Atlas-%s' % app.atlas_release jspec.homepackage = app.home_package jspec.transformation = app.transformation jspec.destinationDBlock = job.outputdata.datasetname if job.outputdata.location: jspec.destinationSE = job.outputdata.location else: jspec.destinationSE = job.backend.site if job.inputdata: jspec.prodDBlock = job.inputdata.dataset[0] else: jspec.prodDBlock = 'NULL' if app.prod_source_label: jspec.prodSourceLabel = app.prod_source_label else: jspec.prodSourceLabel = configPanda['prodSourceLabelRun'] jspec.processingType = configPanda['processingType'] jspec.specialHandling = configPanda['specialHandling'] jspec.computingSite = job.backend.site jspec.cloud = job.backend.requirements.cloud jspec.cmtConfig = app.atlas_cmtconfig if app.dbrelease == 'LATEST': try: latest_dbrelease = getLatestDBReleaseCaching() except: from pandatools import Client latest_dbrelease = Client.getLatestDBRelease() m = re.search('(.*):DBRelease-(.*)\.tar\.gz', latest_dbrelease) if m: self.dbrelease_dataset = m.group(1) self.dbrelease = m.group(2) else: raise ApplicationConfigurationError( None, "Error retrieving LATEST DBRelease. Try setting application.dbrelease manually." ) else: self.dbrelease_dataset = app.dbrelease_dataset self.dbrelease = app.dbrelease jspec.jobParameters = app.job_parameters if self.dbrelease: if self.dbrelease == 'current': jspec.jobParameters += ' --DBRelease=current' else: if jspec.transformation.endswith( "_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --DBRelease=DBRelease-%s.tar.gz' % ( self.dbrelease, ) else: jspec.jobParameters += ' DBRelease=DBRelease-%s.tar.gz' % ( self.dbrelease, ) dbspec = FileSpec() dbspec.lfn = 'DBRelease-%s.tar.gz' % self.dbrelease dbspec.dataset = self.dbrelease_dataset dbspec.prodDBlock = jspec.prodDBlock dbspec.type = 'input' jspec.addFile(dbspec) if job.inputdata: m = re.search('(.*)\.(.*)\.(.*)\.(.*)\.(.*)\.(.*)', job.inputdata.dataset[0]) if not m: logger.error("Error retrieving run number from dataset name") #raise ApplicationConfigurationError(None, "Error retrieving run number from dataset name") runnumber = 105200 else: runnumber = int(m.group(2)) if jspec.transformation.endswith( "_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --runNumber %d' % runnumber else: jspec.jobParameters += ' RunNumber=%d' % runnumber # Output files. randomized_lfns = [] ilfn = 0 for lfn, lfntype in zip(app.output_files, app.output_type): ofspec = FileSpec() if app.randomize_lfns: randomized_lfn = lfn + ( '.%s.%d.%s' % (job.backend.site, int(time.time()), commands.getoutput('uuidgen 2> /dev/null')[:4])) else: randomized_lfn = lfn ofspec.lfn = randomized_lfn randomized_lfns.append(randomized_lfn) ofspec.destinationDBlock = jspec.destinationDBlock ofspec.destinationSE = jspec.destinationSE ofspec.dataset = jspec.destinationDBlock ofspec.type = 'output' jspec.addFile(ofspec) if jspec.transformation.endswith( "_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --output%sFile %s' % ( lfntype, randomized_lfns[ilfn]) else: jspec.jobParameters += ' output%sFile=%s' % ( lfntype, randomized_lfns[ilfn]) ilfn = ilfn + 1 # Input files. if job.inputdata: for guid, lfn, size, checksum, scope in zip( job.inputdata.guids, job.inputdata.names, job.inputdata.sizes, job.inputdata.checksums, job.inputdata.scopes): ifspec = FileSpec() ifspec.lfn = lfn ifspec.GUID = guid ifspec.fsize = size ifspec.md5sum = checksum ifspec.scope = scope ifspec.dataset = jspec.prodDBlock ifspec.prodDBlock = jspec.prodDBlock ifspec.type = 'input' jspec.addFile(ifspec) if app.input_type: itype = app.input_type else: itype = m.group(5) if jspec.transformation.endswith( "_tf.py") or jspec.transformation.endswith("_tf"): jspec.jobParameters += ' --input%sFile %s' % (itype, ','.join( job.inputdata.names)) else: jspec.jobParameters += ' input%sFile=%s' % (itype, ','.join( job.inputdata.names)) # Log files. lfspec = FileSpec() lfspec.lfn = '%s.job.log.tgz' % jspec.jobName lfspec.destinationDBlock = jspec.destinationDBlock lfspec.destinationSE = jspec.destinationSE lfspec.dataset = jspec.destinationDBlock lfspec.type = 'log' jspec.addFile(lfspec) return jspec
def retry(self, JobsetID, newSite=False, newOpts={}, noSubmit=False, ignoreDuplication=True): # get logger tmpLog = PLogger.getPandaLogger() # check proxy self.gridPassPhrase, self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose) # get jobset newJobsetID = -1 jobList = self.getJobIDsWithSetID(JobsetID) if jobList == None: isJobset = False jobList = [JobsetID] else: isJobset = True tmpMsg = "JobsetID=%s is composed of JobID=" % JobsetID for tmpJobID in jobList: tmpMsg += '%s,' % tmpJobID tmpMsg = tmpMsg[:-1] tmpLog.info(tmpMsg) for JobID in jobList: # get job info from local repository localJob = self.getJobInfo(JobID) if localJob == None: tmpLog.warning( "JobID=%s not found in local repository. Synchronization may be needed" % JobID) return None # skip running job if localJob.dbStatus != 'frozen': tmpLog.warning('Cannot retry running jobs') if isJobset: continue else: return # skip already retried if localJob.retryID != '0': if isJobset: tmpLog.info('Skip JobID=%s since already retried by JobID=%s JobsetID=%s' % \ (JobID,localJob.retryID,localJob.retryJobsetID)) continue else: tmpLog.warning('This job was already retried by JobID=%s' % localJob.retryID) return # check status of buildJob if not localJob.buildStatus in ['', 'finished']: tmpMsgStr = 'Cannot retry since status of buildJob %s is %s (!= finished). ' \ % (localJob.PandaID.split(',')[0],localJob.buildStatus) tmpMsgStr += 'Please execute %s with the same input/output datasets (or containers). ' % localJob.jobType tmpMsgStr += 'It will run only on failed/cancelled/unused input files ' tmpMsgStr += 'and append output files to the output dataset container' tmpLog.warning(tmpMsgStr) if isJobset: continue else: return # check opts for newSite if newSite: if not localJob.outDS.endswith('/') and not newOpts.has_key( 'outDS') and not newOpts.has_key('--outDS'): tmpLog.warning( 'You need to specify --outDS in newOpts to retry at new site unless container is used as output' ) return # get list of failed jobs pandaIDs = localJob.PandaID.split(',') statusList = localJob.jobStatus.split(',') jobList = [] for idx in range(len(pandaIDs)): # check status if not statusList[idx] in ['failed', 'cancelled']: continue jobList.append(pandaIDs[idx]) # no failed job if jobList == []: if isJobset: tmpLog.info('Skip JobID=%s since no failed jobs' % JobID) continue else: tmpLog.info('No failed jobs to be retried for JobID=%s' % JobID) return # get full job spec tmpLog.info("Retrying JobID=%s ..." % JobID) tmpLog.info("Getting job info") idxJL = 0 nQuery = 500 pandaJobs = [] while idxJL < len(jobList): # avoid burst query tmpLog.info(" %5s/%s" % (idxJL, len(jobList))) status, oTmp = Client.getFullJobStatus(jobList[idxJL:idxJL + nQuery], verbose=self.verbose) if status != 0: tmpLog.error(status) tmpLog.error(oTmp) tmpLog.error("Cannot get job info from Panda server") return pandaJobs += oTmp idxJL += nQuery time.sleep(1) tmpLog.info(" %5s/%s" % (len(jobList), len(jobList))) # jobdefID newJobdefID = PsubUtils.readJobDefID() # reset some parameters retryJobs = [] retrySite = None retryElement = None retryDestSE = None outDsName = None shadowList = [] for idx in range(len(jobList)): job = pandaJobs[idx] # skip exired if job == None: tmpLog.warning("Could not retry jobs older than 30 days : JobID=%s (PandaID=%s) expired" \ % (JobID,jobList[idxJob])) return # get shadow list if (not ignoreDuplication ) and outDsName == None and job.prodSourceLabel == 'user': # look for dataset for log since it doesn't have suffix even when --individualOutDS is used for tmpFile in job.Files: if tmpFile.type == 'log': outDsName = tmpFile.dataset break # output dataset was not found if outDsName == None: tmpLog.error("Could not get output dataset name for JobID=%s (PandaID=%s)" \ % (JobID,job.PandaID)) return # get files in shadow if outDsName.endswith('/'): shadowList = Client.getFilesInShadowDataset( outDsName, Client.suffixShadow, self.verbose) else: # disable duplication check mainly for old overlay jobs since non-signal files are wrongly skipped #shadowList = Client.getFilesInShadowDatasetOld(outDsName,Client.suffixShadow,self.verbose) pass # unify sitename if retrySite == None: retrySite = job.computingSite retryElement = job.computingElement retryDestSE = job.destinationSE # reset job.jobStatus = None job.commandToPilot = None job.startTime = None job.endTime = None job.attemptNr = 1 + job.attemptNr for attr in job._attributes: if attr.endswith('ErrorCode') or attr.endswith( 'ErrorDiag'): setattr(job, attr, None) job.transExitCode = None job.computingSite = retrySite job.computingElement = retryElement job.destinationSE = retryDestSE job.dispatchDBlock = None job.jobExecutionID = JobID job.jobDefinitionID = newJobdefID job.parentID = job.PandaID if job.jobsetID != ['NULL', None, -1]: job.sourceSite = job.jobsetID job.jobsetID = newJobsetID skipInputList = [] numUsedFiles = 0 for file in job.Files: file.rowID = None if file.type == 'input': # protection against wrong sync which doesn't update buildStatus correctly if file.lfn.endswith( '.lib.tgz') and file.GUID == 'NULL': tmpLog.warning('GUID for %s is unknown. Cannot retry when corresponding buildJob failed' \ % file.lfn) return file.status = 'ready' # check with shadow for non lib.tgz/DBR tmpDbrMatch = re.search('^DBRelease-.*\.tar\.gz$', file.lfn) if tmpDbrMatch == None and not file.lfn.endswith( '.lib.tgz'): if file.lfn in shadowList: skipInputList.append(file) else: numUsedFiles += 1 elif file.type in ('output', 'log'): file.destinationSE = retryDestSE file.destinationDBlock = re.sub( '_sub\d+$', '', file.destinationDBlock) # add retry num if file.dataset.endswith('/'): retryMatch = re.search('_r(\d+)$', file.destinationDBlock) if retryMatch == None: file.destinationDBlock += '_r1' else: tmpDestinationDBlock = re.sub( '_r(\d+)$', '', file.destinationDBlock) file.destinationDBlock = tmpDestinationDBlock + '_r%d' % ( 1 + int(retryMatch.group(1))) # add attempt nr oldName = file.lfn file.lfn = re.sub("\.\d+$", "", file.lfn) file.lfn = "%s.%d" % (file.lfn, job.attemptNr) newName = file.lfn # modify jobParameters job.jobParameters = re.sub("'%s'" % oldName, "'%s'" % newName, job.jobParameters) # look for output in trf oldGenelicName = re.sub('\.\d+$', '', oldName) match = re.search(oldGenelicName + '(\.\d+)*(%20|")', job.jobParameters) if match != None: job.jobParameters = job.jobParameters.replace( match.group(0), newName + match.group(2)) # all files are used by others if numUsedFiles == 0 and skipInputList != []: continue # remove skipped files strSkipped = '' for tmpFile in skipInputList: strSkipped += '%s,' % tmpFile.lfn job.Files.remove(tmpFile) strSkipped = strSkipped[:-1] # modify jobpar if strSkipped != '': optionToSkipFiles = '--skipInputByRetry' if not optionToSkipFiles in job.jobParameters: # just append job.jobParameters += "%s=%s " % (optionToSkipFiles, strSkipped) else: # extract already skipped files tmpMatch = re.search("(%s=[^ ]+)", job.jobParameters) if tmpMatch == None: tmpLog.error("Failed to extract arg of %s for PandaID=%s" \ % (optionToSkipFiles,job.PandaID)) return # replace job.jobParameters = re.sub( tmpMatch.group(1), "%s,%s" % (tmpMatch.group(1), optionToSkipFiles), job.jobParameters) if self.verbose: tmpLog.debug(job.jobParameters) # append retryJobs.append(job) # all input files were or are being used by other jobs if retryJobs == []: tmpLog.info( 'All input files were or are being used by other jobs for the same output. No jobs to be retried. If you need to ignore duplication check (e.g., using the same EVNT file for multiple simulation subjobs), set ignoreDuplication=True. i.e. retry(123,ignoreDuplication=True)' ) return # check voms role if not retryJobs[0].workingGroup in ['NULL', None, '']: # VOMS role was used if not "--workingGroup" in job.metadata: # extract voms roles from metadata match = re.search("--voms( |=)[ \"]*([^ \"]+)", job.metadata) if match != None: vomsRoles = match.group(2) else: vomsRoles = "atlas:/atlas/%s/Role=production" % retryJobs[ 0].workingGroup # regenerate proxy with VOMS roles try: tmpLog.info("Checking proxy role to resubmit %s jobs" % retryJobs[0].workingGroup) self.gridPassPhrase, self.vomsFQAN = PsubUtils.checkGridProxy( self.gridPassPhrase, False, self.verbose, vomsRoles) except: tmpLog.error("Failed to generate a proxy with %s" % vomsRoles) return # check runtime env for new site submission if newSite: if retryJobs[ 0].processingType == 'pathena' or '--useAthenaPackages' in retryJobs[ 0].metadata: from pandatools import AthenaUtils stA, retA = AthenaUtils.getAthenaVer() if not stA: tmpLog.error( "Failed to get Athena rel/cache version in current runtime env" ) return athenaVer = retA['athenaVer'] cacheVer = retA['cacheVer'] nightVer = retA['nightVer'] wrongSetup = False if retryJobs[0].AtlasRelease != 'Atlas-%s' % athenaVer: wrongSetup = True errMsg = "Current Athena version Atlas-%s is inconsitent with the previous submission %s. " % ( athenaVer, retryJobs[0].AtlasRelease) elif retryJobs[ 0].homepackage != 'AnalysisTransforms' + cacheVer + nightVer: wrongSetup = True errMsg = "Current cache version %s is inconsitent with the previous submission. " % cacheVer.replace( '-', '').replace('_', '-') if wrongSetup: errMsg += 'You need to have the same runtime env as before since all job spec need to be re-created to send jobs to a new site. ' errMsg += 'Please setup Athena correctly and restart pbook' tmpLog.error(errMsg) return # test mode if noSubmit: continue # invoke pathena/prun to send job to new site if newSite: tmpLog.info( "Constrcuting job spec again to be sent to another site ..." ) comStat = PsubUtils.execWithModifiedParams( retryJobs, newOpts, self.verbose) if comStat == 0: # update database time.sleep(2) self.sync() else: tmpLog.error("Failed to submit jobs to Panda server") return # register datasets tmpOutDsLocation = Client.PandaSites[ retryJobs[-1].computingSite]['ddm'] addedDataset = [] shadowDSname = None for tmpFile in retryJobs[-1].Files: if tmpFile.type in ['output', 'log' ] and tmpFile.dataset.endswith('/'): # add shadow if shadowDSname == None and tmpFile.type == 'log': shadowDSname = "%s%s" % (tmpFile.destinationDBlock, Client.suffixShadow) Client.addDataset(shadowDSname, self.verbose) # add datasets if not tmpFile.destinationDBlock in addedDataset: # create dataset Client.addDataset(tmpFile.destinationDBlock, self.verbose, location=tmpOutDsLocation) # add to container Client.addDatasetsToContainer( tmpFile.dataset, [tmpFile.destinationDBlock], self.verbose) # append addedDataset.append(tmpFile.destinationDBlock) # submit tmpLog.info("Submitting job ...") status, out = Client.submitJobs(retryJobs, verbose=self.verbose) if out == None or status != 0: tmpLog.error(status) tmpLog.error(out) tmpLog.error("Failed to submit jobs to Panda server") return # update database pandaIDstatus = {} newJobID = None for items in out: # get newJobID if newJobID == None: newJobID = items[1] # check PandaID PandaID = items[0] if PandaID == 'NULL': tmpLog.error( "Panda server returned wrong IDs. It may have a temporary problem" ) return # set newJobsetID if newJobsetID in [None, -1]: newJobsetID = items[2]['jobsetID'] # dummy statuso pandaIDstatus[PandaID] = ('defined', 'NULL') # set retry ID localJob.retryID = newJobID if not newJobsetID in [None, -1, 'NULL']: localJob.retryJobsetID = newJobsetID try: PdbUtils.updateJobDB(localJob, self.verbose) except: tmpLog.error("Failed to set retryID for JobID=%s" % JobID) return # set new paramers newLocalJob = PdbUtils.convertPtoD(retryJobs, pandaIDstatus) newLocalJob.JobID = newJobID if not newJobsetID in [None, -1, 'NULL']: newLocalJob.groupID = newJobsetID newLocalJob.creationTime = datetime.datetime.utcnow() # insert to DB try: PdbUtils.insertJobDB(newLocalJob, self.verbose) except: tmpLog.error("Failed to insert JobID=%s to local repository" % newJobID) return # write new jobdefID PsubUtils.writeJobDefID(newJobID) # done tmpMsg = 'Done. New JobID=%s' % newJobID if not newJobsetID in [None, -1, 'NULL']: tmpMsg += " JobsetID=%s" % newJobsetID tmpLog.info(tmpMsg)
def master_prepare(self, app, appmasterconfig): # PandaTools from pandatools import Client from pandatools import AthenaUtils from taskbuffer.JobSpec import JobSpec from taskbuffer.FileSpec import FileSpec job = app._getParent() logger.debug('AthenaMCPandaRTHandler master_prepare called for %s', job.getFQID('.')) usertag = configDQ2['usertag'] #usertag='user09' nickname = getNickname(allowMissingNickname=True) self.libDataset = '%s.%s.ganga.%s_%d.lib._%06d' % ( usertag, nickname, commands.getoutput('hostname').split('.')[0], int(time.time()), job.id) # self.userprefix='%s.%s.ganga' % (usertag,gridProxy.identity()) sources = 'sources.%s.tar.gz' % commands.getoutput( 'uuidgen 2> /dev/null') self.library = '%s.lib.tgz' % self.libDataset # check DBRelease # if job.backend.dbRelease != '' and job.backend.dbRelease.find(':') == -1: # raise ApplicationConfigurationError(None,"ERROR : invalid argument for backend.dbRelease. Must be 'DatasetName:FileName'") # unpack library logger.debug('Creating source tarball ...') tmpdir = '/tmp/%s' % commands.getoutput('uuidgen 2> /dev/null') os.mkdir(tmpdir) inputbox = [] if os.path.exists(app.transform_archive): # must add a condition on size. inputbox += [File(app.transform_archive)] if app.evgen_job_option: self.evgen_job_option = app.evgen_job_option if os.path.exists(app.evgen_job_option): # locally modified job option file to add to the input sand box inputbox += [File(app.evgen_job_option)] self.evgen_job_option = app.evgen_job_option.split("/")[-1] # add input sandbox files if (job.inputsandbox): for file in job.inputsandbox: inputbox += [file] # add option files for extFile in job.backend.extOutFile: try: shutil.copy(extFile, tmpdir) except IOError: os.makedirs(tmpdir) shutil.copy(extFile, tmpdir) # fill the archive for opt_file in inputbox: try: shutil.copy(opt_file.name, tmpdir) except IOError: os.makedirs(tmpdir) shutil.copy(opt_file.name, tmpdir) # now tar it up again inpw = job.getInputWorkspace() rc, output = commands.getstatusoutput('tar czf %s -C %s .' % (inpw.getPath(sources), tmpdir)) if rc: logger.error('Packing sources failed with status %d', rc) logger.error(output) raise ApplicationConfigurationError(None, 'Packing sources failed.') shutil.rmtree(tmpdir) # upload sources logger.debug('Uploading source tarball ...') try: cwd = os.getcwd() os.chdir(inpw.getPath()) rc, output = Client.putFile(sources) if output != 'True': logger.error('Uploading sources %s failed. Status = %d', sources, rc) logger.error(output) raise ApplicationConfigurationError( None, 'Uploading archive failed') finally: os.chdir(cwd) # Use Panda's brokerage ## if job.inputdata and len(app.sites)>0: ## # update cloud, use inputdata's ## from dq2.info.TiersOfATLAS import whichCloud,ToACache ## inclouds=[] ## for site in app.sites: ## cloudSite=whichCloud(app.sites[0]) ## if cloudSite not in inclouds: ## inclouds.append(cloudSite) ## # now converting inclouds content into proper brokering stuff. ## outclouds=[] ## for cloudSite in inclouds: ## for cloudID, eachCloud in ToACache.dbcloud.iteritems(): ## if cloudSite==eachCloud: ## cloud=cloudID ## outclouds.append(cloud) ## break ## print outclouds ## # finally, matching with user's wishes ## if len(outclouds)>0: ## if not job.backend.requirements.cloud: # no user wish, update ## job.backend.requirements.cloud=outclouds[0] ## else: ## try: ## assert job.backend.requirements.cloud in outclouds ## except: ## raise ApplicationConfigurationError(None,'Input dataset not available in target cloud %s. Please try any of the following %s' % (job.backend.requirements.cloud, str(outclouds))) from GangaPanda.Lib.Panda.Panda import runPandaBrokerage runPandaBrokerage(job) if job.backend.site == 'AUTO': raise ApplicationConfigurationError( None, 'site is still AUTO after brokerage!') # output dataset preparation and registration try: outDsLocation = Client.PandaSites[job.backend.site]['ddm'] except: raise ApplicationConfigurationError( None, "Could not extract output dataset location from job.backend.site value: %s. Aborting" % job.backend.site) if not app.dryrun: for outtype in app.outputpaths.keys(): dset = string.replace(app.outputpaths[outtype], "/", ".") dset = dset[1:] # dataset registration must be done only once. print "registering output dataset %s at %s" % (dset, outDsLocation) try: Client.addDataset(dset, False, location=outDsLocation) dq2_set_dataset_lifetime(dset, location=outDsLocation) except: raise ApplicationConfigurationError( None, "Fail to create output dataset %s. Aborting" % dset) # extend registration to build job lib dataset: print "registering output dataset %s at %s" % (self.libDataset, outDsLocation) try: Client.addDataset(self.libDataset, False, location=outDsLocation) dq2_set_dataset_lifetime(self.libDataset, outDsLocation) except: raise ApplicationConfigurationError( None, "Fail to create output dataset %s. Aborting" % self.libDataset) ### cacheVer = "-AtlasProduction_" + str(app.prod_release) logger.debug("master job submit?") self.outsite = job.backend.site if app.se_name and app.se_name != "none" and not self.outsite: self.outsite = app.se_name # create build job jspec = JobSpec() jspec.jobDefinitionID = job.id jspec.jobName = commands.getoutput('uuidgen 2> /dev/null') jspec.AtlasRelease = 'Atlas-%s' % app.atlas_rel jspec.homepackage = 'AnalysisTransforms' + cacheVer #+nightVer jspec.transformation = '%s/buildJob-00-00-03' % Client.baseURLSUB # common base to Athena and AthenaMC jobs: buildJob is a pilot job which takes care of all inputs for the real jobs (in prepare() jspec.destinationDBlock = self.libDataset jspec.destinationSE = job.backend.site jspec.prodSourceLabel = 'panda' jspec.assignedPriority = 2000 jspec.computingSite = job.backend.site jspec.cloud = job.backend.requirements.cloud # jspec.jobParameters = self.args not known yet jspec.jobParameters = '-o %s' % (self.library) if app.userarea: print app.userarea jspec.jobParameters += ' -i %s' % (os.path.basename(app.userarea)) else: jspec.jobParameters += ' -i %s' % (sources) jspec.cmtConfig = AthenaUtils.getCmtConfig(athenaVer=app.atlas_rel) matchURL = re.search('(http.*://[^/]+)/', Client.baseURLSSL) if matchURL: jspec.jobParameters += ' --sourceURL %s' % matchURL.group(1) fout = FileSpec() fout.lfn = self.library fout.type = 'output' fout.dataset = self.libDataset fout.destinationDBlock = self.libDataset jspec.addFile(fout) flog = FileSpec() flog.lfn = '%s.log.tgz' % self.libDataset flog.type = 'log' flog.dataset = self.libDataset flog.destinationDBlock = self.libDataset jspec.addFile(flog) #print "MASTER JOB DETAILS:",jspec.jobParameters return jspec