def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) if app.getMetadata: logger.info("Adding options to make the summary.xml") inputsandbox.append( FileBuffer( 'summary.py', "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'" )) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase() rep_data = app.uploadedInput.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.jobScriptArchive.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.uploadedInput.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.jobScriptArchive.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) assert isinstance(app.uploadedInput, DiracFile), "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase() rep_data = app.uploadedInput.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) assert isinstance(app.jobScriptArchive, DiracFile), "Failed to upload needed file, aborting submit" rep_data = app.jobScriptArchive.getReplicas() assert rep_data != {}, "Failed to find a replica, aborting submit" return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox']) # add summary.xml outputsandbox += ['summary.xml', '__parsedxmlsummary__'] return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox']) # add summary.xml outputsandbox += ['summary.xml', '__parsedxmlsummary__'] return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ This function prepares the application of a master job during submit. A priori we aren't doing anything with this in Im3ShapeApp but until this is understood I'd rather not remove it Args: app (IApplication): This is the application given in the master job appasterconfig (tuple): This is the configuration which is to prepare the app in the master job # TODO check type and this interface """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ cred_req = app.getJobObject().backend.credential_requirements check_creds(cred_req) inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] # Check a previously uploaded input is there in case of a job copy if isinstance(app.uploadedInput, DiracFile): if app.uploadedInput.getReplicas() == {}: app.uploadedInput = None logger.info("Previously uploaded cmake target missing from Dirac. Uploading it again.") if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) try: assert isinstance(app.uploadedInput, DiracFile) except AssertionError: raise ApplicationPrepareError("Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req)) rep_data = app.uploadedInput.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError("Failed to find a replica of uploaded file, aborting submit") if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) try: assert isinstance(app.jobScriptArchive, DiracFile) except AssertionError: raise ApplicationPrepareError("Failed to upload needed file, aborting submit") rep_data = app.jobScriptArchive.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError("Failed to find a replica, aborting submit") #Create a replica of the job and scripts files replicateJobFile(app.jobScriptArchive) replicateJobFile(app.uploadedInput) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ This function prepares the application of a master job during submit. A priori we aren't doing anything with this in Im3ShapeApp but until this is understood I'd rather not remove it Args: app (IApplication): This is the application given in the master job appasterconfig (tuple): This is the configuration which is to prepare the app in the master job # TODO check type and this interface """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ["inputsandbox"]) # add summary.xml outputsandbox += ["summary.xml", "__parsedxmlsummary__"] logger.debug("Master Prepare LHCbGaudiDiracRunTimeHandler") return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if type(app.exe) == File: input_dir = app.getJobObject().getInputWorkspace().getPath() exefile = os.path.join(input_dir, os.path.basename(app.exe.name)) if not os.path.exists(exefile): msg = 'Executable: "%s" must exist!' % str(exefile) raise ApplicationConfigurationError(None, msg) os.system('chmod +x %s' % exefile) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiRun): This application is only expected to handle GaudiRun Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ WarnUsers() inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if type(app.exe) == File: input_dir = app.getJobObject().getInputWorkspace().getPath() exefile = os.path.join(input_dir, os.path.basename(app.exe.name)) if not os.path.exists(exefile): msg = 'Executable: "%s" must exist!' % str(exefile) raise ApplicationConfigurationError(None, msg) os.system('chmod +x %s' % exefile) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): logger.debug("Master Prepare") inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig, ['inputsandbox']) # add summary.xml outputsandbox += ['summary.xml', '__parsedxmlsummary__'] logger.debug("Master Prepare LHCbGaudiDiracRunTimeHandler") return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) if type(app.exe) == File: exefile = os.path.join(get_share_path(app), os.path.basename(app.exe.name)) if not os.path.exists(exefile): msg = 'Executable must exist!' raise ApplicationConfigurationError(None, msg) os.system('chmod +x %s' % exefile) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) # check file is set OK if not app.script.name: msg = 'Root.script.name must be set.' raise ApplicationConfigurationError(msg) sharedir_scriptpath = os.path.join(get_share_path(app), os.path.basename(app.script.name)) if not os.path.exists(sharedir_scriptpath): msg = 'Script must exist!' raise ApplicationConfigurationError(msg) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) # check file is set OK if not app.script.name: msg = 'Root.script.name must be set.' raise ApplicationConfigurationError(None, msg) sharedir_scriptpath = os.path.join(get_share_path(app), os.path.basename(app.script.name)) if not os.path.exists(sharedir_scriptpath): msg = 'Script must exist!' raise ApplicationConfigurationError(None, msg) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig) if isinstance(app.jobScriptArchive, LocalFile): app.jobScriptArchive = None generateJobScripts(app, appendJobScripts=True) scriptArchive = os.path.join(app.jobScriptArchive.localDir, app.jobScriptArchive.namePattern) inputsandbox.append(File(name=scriptArchive)) if app.getMetadata: logger.info("Adding options to make the summary.xml") inputsandbox.append(FileBuffer('summary.py', "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'")) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare(app, appmasterconfig, ['inputsandbox']) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig, ['inputsandbox']) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))
def master_prepare(self, app, appmasterconfig): """ Prepare the RTHandler for the master job so that applications to be submitted Args: app (GaudiExec): This application is only expected to handle GaudiExec Applications here appmasterconfig (unknown): Output passed from the application master configuration call """ cred_req = app.getJobObject().backend.credential_requirements check_creds(cred_req) inputsandbox, outputsandbox = master_sandbox_prepare( app, appmasterconfig) # If we are getting the metadata we need to make sure the summary.xml is added to the output sandbox if not there already. if app.getMetadata and not 'summary.xml' in outputsandbox: outputsandbox += ['summary.xml'] # Check a previously uploaded input is there in case of a job copy if isinstance(app.uploadedInput, DiracFile): if app.uploadedInput.getReplicas() == {}: app.uploadedInput = None logger.info( "Previously uploaded cmake target missing from Dirac. Uploading it again." ) if not isinstance(app.uploadedInput, DiracFile): generateDiracInput(app) try: assert isinstance(app.uploadedInput, DiracFile) except AssertionError: raise ApplicationPrepareError( "Failed to upload needed file, aborting submit. Tried to upload to: %s\nIf your Ganga installation is not at CERN your username may be trying to create a non-existent LFN. Try setting the 'DIRAC' configuration 'DiracLFNBase' to your grid user path.\n" % DiracFile.diracLFNBase(cred_req)) rep_data = app.uploadedInput.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError( "Failed to find a replica of uploaded file, aborting submit") if isinstance(app.jobScriptArchive, (DiracFile, LocalFile)): app.jobScriptArchive = None generateDiracScripts(app) try: assert isinstance(app.jobScriptArchive, DiracFile) except AssertionError: raise ApplicationPrepareError( "Failed to upload needed file, aborting submit") rep_data = app.jobScriptArchive.getReplicas() try: assert rep_data != {} except AssertionError: raise ApplicationPrepareError( "Failed to find a replica, aborting submit") #Create a replica of the job and scripts files replicateJobFile(app.jobScriptArchive) replicateJobFile(app.uploadedInput) return StandardJobConfig(inputbox=unique(inputsandbox), outputbox=unique(outputsandbox))