def getInputFileDir(job): """ Return the LFN remote dirname for this job """ return os.path.join( DiracFile.diracLFNBase(job.backend.credential_requirements), 'GangaJob_%s/InputFiles' % job.fqid)
def getDiracFiles(): import os from GangaDirac.Lib.Files.DiracFile import DiracFile from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns' logger.info('Creating list, this can take a while if you have a large number of SE files, please wait...') execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None) g = GangaList() with open(filename[1:], 'r') as lfnlist: lfnlist.seek(0) g.extend((DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines())) return addProxy(g)
def getDiracFiles(): import os from GangaDirac.Lib.Files.DiracFile import DiracFile from Ganga.GPIDev.Lib.GangaList.GangaList import GangaList filename = DiracFile.diracLFNBase().replace('/', '-') + '.lfns' logger.info( 'Creating list, this can take a while if you have a large number of SE files, please wait...' ) execute('dirac-dms-user-lfns &> /dev/null', shell=True, timeout=None) g = GangaList() with open(filename[1:], 'r') as lfnlist: lfnlist.seek(0) g.extend( (DiracFile(lfn='%s' % lfn.strip()) for lfn in lfnlist.readlines())) return addProxy(g)
def generateDiracInput(app): """ Construct a DIRAC input which must be unique to each job to have unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiRun): This expects a GaudiRun app to be passed so that the constructed """ input_files, input_folders = collectPreparedFiles(app) job = app.getJobObject() if input_folders: raise ApplicationConfigurationError(None, 'Prepared folders not supported yet, please fix this in future') else: prep_dir = app.getSharedPath() add_timeStampFile(prep_dir) prep_file = prep_dir + '.tgz' compressed_file = os.path.join(tempfile.gettempdir(), '__'+os.path.basename(prep_file)) wnScript = generateWNScript(prepareCommand(app), job) script_name = os.path.join(tempfile.gettempdir(), wnScript.name) wnScript.create(script_name) with tarfile.open(compressed_file, "w:gz") as tar_file: for name in input_files: tar_file.add(name, arcname=os.path.basename(name)) tar_file.add(script_name, arcname=os.path.basename(script_name)) shutil.move(compressed_file, prep_dir) new_df = DiracFile(namePattern=os.path.basename(compressed_file), localDir=app.getSharedPath()) random_SE = random.choice(getConfig('DIRAC')['allDiracSE']) logger.info("new File: %s" % new_df) new_lfn = os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid, os.path.basename(compressed_file)) new_df.put(uploadSE=random_SE, lfn=new_lfn) app.uploadedInput = new_df
def getInputFileDir(job): """ Return the LFN remote dirname for this job """ return os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid)
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ cred_req = app.getJobObject().backend.credential_requirements check_creds(cred_req) inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] # Check a previously uploaded input is there in case of a job copy if isinstance(app.uploadedInput, DiracFile): if app.uploadedInput.getReplicas() == {}: app.uploadedInput = None logger.info( "Previously uploaded cmake target missing from Dirac. Uploading it again." ) if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) try: assert isinstance(app.uploadedInput, DiracFile) except AssertionError: raise ApplicationPrepareError( "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req)) rep_data = app.uploadedInput.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError( "Failed to find a replica of uploaded file, aborting submit") if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) try: assert isinstance(app.jobScriptArchive, DiracFile) except AssertionError: raise ApplicationPrepareError( "Failed to upload needed file, aborting submit") rep_data = app.jobScriptArchive.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError( "Failed to find a replica, aborting submit") #Create a replica of the job and scripts files replicateJobFile(app.jobScriptArchive) replicateJobFile(app.uploadedInput) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ cred_req = app.getJobObject().backend.credential_requirements check_creds(cred_req) inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] # Check a previously uploaded input is there in case of a job copy if isinstance(app.uploadedInput, DiracFile): if app.uploadedInput.getReplicas() == {}: app.uploadedInput = None logger.info("Previously uploaded cmake target missing from Dirac. Uploading it again.") if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) try: assert isinstance(app.uploadedInput, DiracFile) except AssertionError: raise ApplicationPrepareError("Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req)) rep_data = app.uploadedInput.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError("Failed to find a replica of uploaded file, aborting submit") if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) try: assert isinstance(app.jobScriptArchive, DiracFile) except AssertionError: raise ApplicationPrepareError("Failed to upload needed file, aborting submit") rep_data = app.jobScriptArchive.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError("Failed to find a replica, aborting submit") #Create a replica of the job and scripts files replicateJobFile(app.jobScriptArchive) replicateJobFile(app.uploadedInput) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def getInputFileDir(job): """ Return the LFN remote dirname for this job """ return os.path.join(DiracFile.diracLFNBase(job.backend.credential_requirements), 'GangaJob_%s/InputFiles' % job.fqid)
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase() rep_data = app.uploadedInput.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.jobScriptArchive.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def getInputFileDir(job): """ Return the LFN remote dirname for this job """ return os.path.join(DiracFile.diracLFNBase(), 'GangaInputFile/Job_%s' % job.fqid)
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase() rep_data = app.uploadedInput.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.jobScriptArchive.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))