def createFileMetadata(outFiles, job, outsDict, dsname, datasetDict, sitename): """ create the metadata for the output + log files """ ec = 0 # get/assign guids to the output files if outFiles: tolog("outFiles=%s"%str(outFiles)) if not pUtil.isBuildJob(outFiles): ec, job.pilotErrorDiag, job.outFilesGuids = RunJobUtilities.getOutFilesGuids(job.outFiles, job.workdir) if ec: # missing PoolFileCatalog (only error code from getOutFilesGuids) return ec, job, None else: tolog("Build job - do not use PoolFileCatalog to get guid (generated)") else: tolog("This job has no output files") # get the file sizes and checksums for the local output files # WARNING: any errors are lost if occur in getOutputFileInfo() ec, pilotErrorDiag, fsize, checksum = pUtil.getOutputFileInfo(list(outFiles), getChecksumCommand(), skiplog=True, logFile=job.logFile) if ec != 0: tolog("!!FAILED!!2999!! %s" % (pilotErrorDiag)) failJob(job.result[1], ec, job, pilotserver, pilotport, pilotErrorDiag=pilotErrorDiag) if logguid: guid = logguid else: guid = job.tarFileGuid # create preliminary metadata (no metadata yet about log file - added later in pilot.py) _fname = "%s/metadata-%d.xml" % (job.workdir, job.jobId) try: _status = pUtil.PFCxml(job.experiment, _fname, list(job.outFiles), fguids=job.outFilesGuids, fntag="lfn", alog=job.logFile, alogguid=guid,\ fsize=fsize, checksum=checksum, analJob=analJob) except Exception, e: pilotErrorDiag = "PFCxml failed due to problematic XML: %s" % (e) tolog("!!WARNING!!1113!! %s" % (pilotErrorDiag)) failJob(job.result[1], error.ERR_MISSINGGUID, job, pilotserver, pilotport, pilotErrorDiag=pilotErrorDiag)
def getAnalysisRunCommand(self, job, jobSite, trfName): """ Get the run command for analysis jobs """ # The run command is used to setup up runAthena/runGen from RunJobUtilities import updateCopysetups ec = 0 pilotErrorDiag = "" run_command = "" # get the queuedata info # (directAccess info is stored in the copysetup variable) region = readpar('region') # get relevant file transfer info dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\ self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles)) # extract the setup file from copysetup (and verify that it exists) _copysetup = self.getSetupFromCopysetup(copysetup) if _copysetup != "" and os.path.exists(_copysetup): run_command = 'source %s;' % (_copysetup) # add the user proxy if os.environ.has_key('X509_USER_PROXY'): run_command += 'export X509_USER_PROXY=%s;' % os.environ['X509_USER_PROXY'] else: tolog("Could not add user proxy to the run command (proxy does not exist)") # set up analysis trf run_command += './%s %s' % (trfName, job.jobPars) #PN # job.jobPars += ' --accessmode=filestager' # add options for file stager if necessary if dInfo: # sort out when directIn and useFileStager options should be used if useDirectAccess and '--directIn' not in job.jobPars and '--directIn' not in run_command: run_command += ' --directIn' if useFileStager and '--useFileStager' not in job.jobPars: run_command += ' --useFileStager' # old style copysetups will contain oldPrefix and newPrefix needed for the old style remote I/O if oldPrefix != "" and newPrefix != "": run_command += ' --oldPrefix "%s" --newPrefix %s' % (oldPrefix, newPrefix) else: # --directIn should be used in combination with --usePFCTurl, but not --old/newPrefix and --lfcHost if usePFCTurl: run_command += ' --usePFCTurl' if job.transferType == 'direct': # update the copysetup # transferType is only needed if copysetup does not contain remote I/O info updateCopysetups(run_command, transferType=job.transferType, useCT=False, directIn=useDirectAccess, useFileStager=useFileStager) # add options for file stager if necessary (ignore if transferType = direct) if "accessmode" in job.jobPars and job.transferType != 'direct': accessmode_useCT = None accessmode_useFileStager = None accessmode_directIn = None _accessmode_dic = { "--accessmode=copy":["copy-to-scratch mode", ""], "--accessmode=direct":["direct access mode", " --directIn"], "--accessmode=filestager":["direct access / file stager mode", " --directIn --useFileStager"]} # update run_command according to jobPars for _mode in _accessmode_dic.keys(): if _mode in job.jobPars: # any accessmode set in jobPars should overrule schedconfig tolog("Enforcing %s" % (_accessmode_dic[_mode][0])) if _mode == "--accessmode=copy": # make sure direct access and file stager get turned off usePFCTurl = False accessmode_useCT = True accessmode_directIn = False accessmode_useFileStager = False elif _mode == "--accessmode=direct": # make sure copy-to-scratch and file stager get turned off usePFCTurl = True accessmode_useCT = False accessmode_directIn = True accessmode_useFileStager = False else: # make sure file stager gets turned on usePFCTurl = False accessmode_useCT = False accessmode_directIn = True accessmode_useFileStager = True # update run_command (do not send the accessmode switch to runAthena) run_command += _accessmode_dic[_mode][1] if _mode in run_command: run_command = run_command.replace(_mode, "") if "directIn" in run_command and not dInfo: if not usePFCTurl: usePFCTurl = True tolog("WARNING: accessmode mode specified directIn but direct access mode is not specified in copysetup (will attempt to create TURL based PFC later)") if not "usePFCTurl" in run_command: run_command += ' --usePFCTurl' # need to add proxy if not there already if ("--directIn" in run_command or "--useFileStager" in run_command) and not "export X509_USER_PROXY" in run_command: if os.environ.has_key('X509_USER_PROXY'): run_command = run_command.replace("./%s" % (trfName), "export X509_USER_PROXY=%s;./%s" % (os.environ['X509_USER_PROXY'], trfName)) else: tolog("Did not add user proxy to the run command (proxy does not exist)") # add the lfcHost if not there already if not "--lfcHost" in run_command and lfcHost != "": run_command += " --lfcHost %s" % (lfcHost) # update the copysetup updateCopysetups(run_command, transferType=None, useCT=accessmode_useCT, directIn=accessmode_directIn, useFileStager=accessmode_useFileStager) # add guids and lfc host when needed if region == 'US' and jobSite.dq2url != "" and lfcHost == "": # old style run_command += ' -u %s' % (jobSite.dq2url) else: if lfcHost != "": # get the correct guids list (with only the direct access files) if not isBuildJob(job.outFiles): _guids = self.getGuidsFromJobPars(job.jobPars, job.inFiles, job.inFilesGuids) # only add the lfcHost if --usePFCTurl is not specified if usePFCTurl: run_command += ' --inputGUIDs \"%s\"' % (str(_guids)) else: if not "--lfcHost" in run_command: run_command += ' --lfcHost %s' % (lfcHost) run_command += ' --inputGUIDs \"%s\"' % (str(_guids)) else: if not usePFCTurl and not "--lfcHost" in run_command: run_command += ' --lfcHost %s' % (lfcHost) # if both direct access and the accessmode loop added a directIn switch, remove the first one from the string if run_command.count("directIn") > 1: run_command = run_command.replace("--directIn", "", 1) # make sure that the site supports direct access / file stager # if ("--directIn" in run_command or "--useFileStager" in run_command) and readpar('allowdirectaccess').lower() != "true": # ec = self.__error.ERR_DAFSNOTALLOWED # pilotErrorDiag = "Site does not allow requested direct access / file stager (payload will fail)" # tolog("!!WARNING!!1234!! %s" % (pilotErrorDiag)) return ec, pilotErrorDiag, run_command
# update the current file states updateFileStates(outs, runJob.getParentWorkDir(), job.jobId, mode="file_state", state="created") dumpFileStates(runJob.getParentWorkDir(), job.jobId) # create xml string to pass to dispatcher for atlas jobs outputFileInfo = {} if outs or (job.logFile and job.logFile != ''): # get the datasets for the output files dsname, datasetDict = runJob.getDatasets(job) # re-create the metadata.xml file, putting guids of ALL output files into it. # output files that miss guids from the job itself will get guids in PFCxml function # first rename and copy the trf metadata file for non-build jobs if not pUtil.isBuildJob(outs): runJob.moveTrfMetadata(job.workdir, job.jobId) # create the metadata for the output + log files ec, job, outputFileInfo = runJob.createFileMetadata(list(outs), job, outsDict, dsname, datasetDict, jobSite.sitename, analysisJob=analysisJob) if ec: runJob.failJob(0, ec, job, pilotErrorDiag=job.pilotErrorDiag) # move output files from workdir to local DDM area finalUpdateDone = False if outs: tolog("Setting stage-out state until all output files have been copied") job.setState(["stageout", 0, 0]) rt = RunJobUtilities.updatePilotServer(job, runJob.getPilotServer(), runJob.getPilotPort()) # stage-out output files
job.jobId, mode="file_state", state="created") dumpFileStates(runJob.getParentWorkDir(), job.jobId) # create xml string to pass to dispatcher for atlas jobs outputFileInfo = {} if outs or (job.logFile and job.logFile != ''): # get the datasets for the output files dsname, datasetDict = runJob.getDatasets(job) # re-create the metadata.xml file, putting guids of ALL output files into it. # output files that miss guids from the job itself will get guids in PFCxml function # first rename and copy the trf metadata file for non-build jobs if not pUtil.isBuildJob(outs): runJob.moveTrfMetadata(job.workdir, job.jobId) # create the metadata for the output + log files ec, job, outputFileInfo = runJob.createFileMetadata( list(outs), job, outsDict, dsname, datasetDict, jobSite.sitename, analysisJob=analysisJob) if ec: runJob.failJob(0, ec, job, pilotErrorDiag=job.pilotErrorDiag) # move output files from workdir to local DDM area
def getCMSRunCommand(self, job, jobSite, trfName): from RunJobUtilities import updateCopysetups ec = 0 pilotErrorDiag = "" run_command = "" # get relevant file transfer info dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\ self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles)) # extract the setup file from copysetup (and verify that it exists) _copysetup = self.getSetupFromCopysetup(copysetup) tolog("copysetup = %s" % _copysetup) if _copysetup != "" and os.path.exists(_copysetup): run_command = 'source %s; ' % (_copysetup) # add the user proxy if os.environ.has_key('X509_USER_PROXY'): run_command += 'export X509_USER_PROXY=%s; ' % os.environ['X509_USER_PROXY'] else: tolog("Could not add user proxy to the run command (proxy does not exist)") """ strpars = job.jobPars cmdopt = shlex.split(strpars) parser = PassThroughOptionParser() parser.add_option('-a',\ dest='a',\ type='string') parser.add_option('-o',\ dest='o',\ type='string') parser.add_option('--inputFile',\ dest='inputFile',\ type='string') parser.add_option('--sourceURL',\ dest='sourceURL',\ type='string') parser.add_option('--jobNumber',\ dest='jobNumber',\ type='string') parser.add_option('--cmsswVersion',\ dest='cmsswVersion',\ type='string') parser.add_option('--scramArch',\ dest='scramArch',\ type='string') parser.add_option('--runAndLumis',\ dest='runAndLumis',\ type='string') (options,args) = parser.parse_args(cmdopt) paramsstring = '-a %s ' % options.a paramsstring += '--sourceURL %s ' % options.sourceURL paramsstring += '--jobNumber=%s ' % options.jobNumber paramsstring += '--cmsswVersion=%s ' % options.cmsswVersion paramsstring += '--scramArch=%s ' % options.scramArch paramsstring += "--inputFile='%s' " % options.inputFile paramsstring += "--runAndLumis='%s' " % options.runAndLumis paramsstring += '-o "%s" ' % options.o tolog("paramsstring = %s" % paramsstring) """ run_command += './%s %s' % (trfName, job.jobPars) return ec, pilotErrorDiag, run_command
tolog("Moved %d/%d output file(s) to: %s" % (nr_moved, len(remaining_files), job.datadir)) # remove all successfully copied files from the local directory nr_removed = 0 for _file in moved_files_list: try: os.system("rm %s" % (_file)) except OSError, e: tolog("!!WARNING!!3000!! Failed to remove output file: %s, %s" % (_file, e)) else: nr_removed += 1 tolog("Removed %d output file(s) from local dir" % (nr_removed)) # copy the PoolFileCatalog.xml for non build jobs if not pUtil.isBuildJob(remaining_files): _fname = os.path.join(job.workdir, "PoolFileCatalog.xml") tolog("Copying %s to %s" % (_fname, job.datadir)) try: copy2(_fname, job.datadir) except Exception, e: tolog("!!WARNING!!3000!! Could not copy PoolFileCatalog.xml to data dir - expect ddm Adder problems during job recovery") # remove all remaining output files from the work directory # (a successfully copied file should already have been removed by the Mover) rem = False for inf in job.outFiles: if inf and inf != 'NULL' and os.path.isfile("%s/%s" % (job.workdir, inf)): # non-empty string and not NULL try: os.remove("%s/%s" % (job.workdir, inf)) except Exception,e:
def getAnalysisRunCommand(self, job, jobSite, trfName): """ Get the run command for analysis jobs """ # The run command is used to setup up runAthena/runGen from RunJobUtilities import updateCopysetups ec = 0 pilotErrorDiag = "" run_command = "" tolog("getAnalysisRunCommand called from Experiment") # get the queuedata info # (directAccess info is stored in the copysetup variable) # get relevant file transfer info dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\ self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles)) # extract the setup file from copysetup (and verify that it exists) _copysetup = self.getSetupFromCopysetup(copysetup) if _copysetup != "" and os.path.exists(_copysetup): run_command = 'source %s;' % (_copysetup) # add the user proxy if os.environ.has_key('X509_USER_PROXY'): run_command += 'export X509_USER_PROXY=%s;' % os.environ[ 'X509_USER_PROXY'] else: tolog( "Could not add user proxy to the run command (proxy does not exist)" ) # set up analysis trf run_command += './%s %s' % (trfName, job.jobPars) # add options for file stager if necessary if dInfo: # in case of forced usePFCTurl if usePFCTurl and not '--usePFCTurl' in run_command: oldPrefix = "" newPrefix = "" run_command += ' --usePFCTurl' tolog("reset old/newPrefix (forced TURL mode (1))") # sort out when directIn and useFileStager options should be used if useDirectAccess and '--directIn' not in job.jobPars and '--directIn' not in run_command: run_command += ' --directIn' if useFileStager and '--useFileStager' not in job.jobPars: run_command += ' --useFileStager' # old style copysetups will contain oldPrefix and newPrefix needed for the old style remote I/O if oldPrefix != "" and newPrefix != "": run_command += ' --oldPrefix "%s" --newPrefix %s' % (oldPrefix, newPrefix) else: # --directIn should be used in combination with --usePFCTurl, but not --old/newPrefix and --lfcHost if usePFCTurl and not '--usePFCTurl' in run_command: run_command += ' --usePFCTurl' if job.transferType == 'direct': # update the copysetup # transferType is only needed if copysetup does not contain remote I/O info updateCopysetups(run_command, transferType=job.transferType, useCT=False, directIn=useDirectAccess, useFileStager=useFileStager) # add options for file stager if necessary (ignore if transferType = direct) if "accessmode" in job.jobPars and job.transferType != 'direct': accessmode_useCT = None accessmode_useFileStager = None accessmode_directIn = None _accessmode_dic = { "--accessmode=copy": ["copy-to-scratch mode", ""], "--accessmode=direct": ["direct access mode", " --directIn"], "--accessmode=filestager": [ "direct access / file stager mode", " --directIn --useFileStager" ] } # update run_command according to jobPars for _mode in _accessmode_dic.keys(): if _mode in job.jobPars: # any accessmode set in jobPars should overrule schedconfig tolog("Enforcing %s" % (_accessmode_dic[_mode][0])) if _mode == "--accessmode=copy": # make sure direct access and file stager get turned off usePFCTurl = False accessmode_useCT = True accessmode_directIn = False accessmode_useFileStager = False elif _mode == "--accessmode=direct": # make sure copy-to-scratch and file stager get turned off usePFCTurl = True accessmode_useCT = False accessmode_directIn = True accessmode_useFileStager = False else: # make sure file stager gets turned on usePFCTurl = False accessmode_useCT = False accessmode_directIn = True accessmode_useFileStager = True # update run_command (do not send the accessmode switch to runAthena) run_command += _accessmode_dic[_mode][1] if _mode in run_command: run_command = run_command.replace(_mode, "") if "directIn" in run_command and not dInfo: if not usePFCTurl: usePFCTurl = True tolog( "WARNING: accessmode mode specified directIn but direct access mode is not specified in copysetup (will attempt to create TURL based PFC later)" ) if not "usePFCTurl" in run_command: run_command += ' --usePFCTurl' # need to add proxy if not there already if ("--directIn" in run_command or "--useFileStager" in run_command ) and not "export X509_USER_PROXY" in run_command: if os.environ.has_key('X509_USER_PROXY'): run_command = run_command.replace( "./%s" % (trfName), "export X509_USER_PROXY=%s;./%s" % (os.environ['X509_USER_PROXY'], trfName)) else: tolog( "Did not add user proxy to the run command (proxy does not exist)" ) # add the lfcHost if not there already if not "--lfcHost" in run_command and lfcHost != "": run_command += " --lfcHost %s" % (lfcHost) # update the copysetup updateCopysetups(run_command, transferType=None, useCT=accessmode_useCT, directIn=accessmode_directIn, useFileStager=accessmode_useFileStager) # add guids and lfc host when needed if lfcHost != "": # get the correct guids list (with only the direct access files) if not isBuildJob(job.outFiles): _guids = self.getGuidsFromJobPars(job.jobPars, job.inFiles, job.inFilesGuids) # only add the lfcHost if --usePFCTurl is not specified if usePFCTurl: run_command += ' --inputGUIDs \"%s\"' % (str(_guids)) else: if not "--lfcHost" in run_command: run_command += ' --lfcHost %s' % (lfcHost) run_command += ' --inputGUIDs \"%s\"' % (str(_guids)) else: if not usePFCTurl and not "--lfcHost" in run_command: run_command += ' --lfcHost %s' % (lfcHost) # if both direct access and the accessmode loop added a directIn switch, remove the first one from the string if run_command.count("directIn") > 1: run_command = run_command.replace("--directIn", "", 1) # make sure that the site supports direct access / file stager # if ("--directIn" in run_command or "--useFileStager" in run_command) and readpar('allowdirectaccess').lower() != "true": # ec = self.__error.ERR_DAFSNOTALLOWED # pilotErrorDiag = "Site does not allow requested direct access / file stager (payload will fail)" # tolog("!!WARNING!!1234!! %s" % (pilotErrorDiag)) return ec, pilotErrorDiag, run_command
def getCMSRunCommand(self, job, jobSite, trfName): from RunJobUtilities import updateCopysetups ec = 0 pilotErrorDiag = "" run_command = "" # get relevant file transfer info dInfo, useCopyTool, useDirectAccess, useFileStager, oldPrefix, newPrefix, copysetup, usePFCTurl, lfcHost =\ self.getFileTransferInfo(job.transferType, isBuildJob(job.outFiles)) # extract the setup file from copysetup (and verify that it exists) _copysetup = "" #self.getSetupFromCopysetup(copysetup) tolog("copysetup = %s" % _copysetup) if _copysetup != "" and os.path.exists(_copysetup): run_command = 'source %s; ' % (_copysetup) # add the user proxy if os.environ.has_key('X509_USER_PROXY'): run_command += 'export X509_USER_PROXY=%s; ' % os.environ['X509_USER_PROXY'] else: tolog("Could not add user proxy to the run command (proxy does not exist)") """ strpars = job.jobPars cmdopt = shlex.split(strpars) parser = PassThroughOptionParser() parser.add_option('-a',\ dest='a',\ type='string') parser.add_option('-o',\ dest='o',\ type='string') parser.add_option('--inputFile',\ dest='inputFile',\ type='string') parser.add_option('--sourceURL',\ dest='sourceURL',\ type='string') parser.add_option('--jobNumber',\ dest='jobNumber',\ type='string') parser.add_option('--cmsswVersion',\ dest='cmsswVersion',\ type='string') parser.add_option('--scramArch',\ dest='scramArch',\ type='string') parser.add_option('--runAndLumis',\ dest='runAndLumis',\ type='string') (options,args) = parser.parse_args(cmdopt) paramsstring = '-a %s ' % options.a paramsstring += '--sourceURL %s ' % options.sourceURL paramsstring += '--jobNumber=%s ' % options.jobNumber paramsstring += '--cmsswVersion=%s ' % options.cmsswVersion paramsstring += '--scramArch=%s ' % options.scramArch paramsstring += "--inputFile='%s' " % options.inputFile paramsstring += "--runAndLumis='%s' " % options.runAndLumis paramsstring += '-o "%s" ' % options.o tolog("paramsstring = %s" % paramsstring) """ run_command += './%s %s' % (trfName, job.jobPars) return ec, pilotErrorDiag, run_command