def prepare(self, force=False): super(Ostap, self).prepare(force) self._check_inputs() share_dir = os.path.join ( expandfilename ( getConfig('Configuration')['gangadir'] ) , 'shared' , getConfig('Configuration')['user'] , self.is_prepared.name ) input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name ) input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name ) fillPackedSandbox ( self.scripts , input_sandbox_tar ) gzipFile ( input_sandbox_tar , input_sandbox_tgz , True ) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() logger.debug("Finished Preparing Application in %s" % share_dir)
def prepare(self, force=False): super(BenderScript, self).prepare(force) self._check_inputs() share_dir = os.path.join ( expandfilename ( getConfig('Configuration')['gangadir'] ) , 'shared' , getConfig('Configuration')['user'] , self.is_prepared.name ) input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name ) input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name ) fillPackedSandbox ( self.scripts + self.imports , input_sandbox_tar ) gzipFile ( input_sandbox_tar , input_sandbox_tgz , True ) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() logger.debug("Finished Preparing Application in %s" % share_dir)
def generateJobScripts(app, appendJobScripts): """ Construct a DIRAC scripts which must be unique to each job to have unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running) """ job = app.getJobObject() if not job.master: rjobs = job.subjobs or [job] else: rjobs = [job] tmp_dir = tempfile.gettempdir() # First create the extraOpts files needed 1 per subjob for this_job in rjobs: logger.debug("RTHandler Making Scripts: %s" % this_job.fqid) this_job.application.constructExtraFiles(this_job) if not job.master and job.subjobs: for sj in rjobs: sj.application.jobScriptArchive = sj.master.application.jobScriptArchive master_job = job.master or job # Now lets get the name of this tar file scriptArchive = os.path.join( master_job.application.jobScriptArchive.localDir, master_job.application.jobScriptArchive.namePattern) if appendJobScripts: # Now lets add the Job scripts to this archive and potentially the extra options to generate the summary.xml with tarfile.open(scriptArchive, 'a') as tar_file: if app.getMetadata: summaryScript = "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'" summaryPath = os.path.join(job.getInputWorkspace().getPath(), 'summary.py') summaryFile = FileBuffer(summaryPath, summaryScript) summaryFile.create() tar_file.add(summaryPath, arcname='summary.py') for this_job in rjobs: this_app = this_job.application wnScript = generateWNScript(prepareCommand(this_app), this_app) this_script = os.path.join(tmp_dir, wnScript.name) wnScript.create(this_script) tar_file.add(this_script, arcname=os.path.join(wnScript.subdir, wnScript.name)) os.unlink(this_script) gzipFile(scriptArchive, scriptArchive + '.gz', True) app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def prepare(self, force=False): super(GaudiPython, self).prepare(force) self._check_inputs() share_dir = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], self.is_prepared.name) fillPackedSandbox(self.script, os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name)) gzipFile(os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name), os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare()
def generateJobScripts(app, appendJobScripts): """ Construct a DIRAC scripts which must be unique to each job to have unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running) """ job = app.getJobObject() if not job.master: rjobs = job.subjobs or [job] else: rjobs = [job] tmp_dir = tempfile.gettempdir() # First create the extraOpts files needed 1 per subjob for this_job in rjobs: logger.debug("RTHandler Making Scripts: %s" % this_job.fqid) this_job.application.constructExtraFiles(this_job) if not job.master and job.subjobs: for sj in rjobs: sj.application.jobScriptArchive = sj.master.application.jobScriptArchive master_job = job.master or job # Now lets get the name of this tar file scriptArchive = os.path.join(master_job.application.jobScriptArchive.localDir, master_job.application.jobScriptArchive.namePattern) if appendJobScripts: # Now lets add the Job scripts to this archive and potentially the extra options to generate the summary.xml with tarfile.open(scriptArchive, 'a') as tar_file: if app.getMetadata: summaryScript = "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'" summaryPath = os.path.join(job.getInputWorkspace().getPath(), 'summary.py') summaryFile = FileBuffer(summaryPath, summaryScript) summaryFile.create() tar_file.add(summaryPath, arcname = 'summary.py') for this_job in rjobs: this_app = this_job.application wnScript = generateWNScript(prepareCommand(this_app), this_app) this_script = os.path.join(tmp_dir, wnScript.name) wnScript.create(this_script) tar_file.add(this_script, arcname=os.path.join(wnScript.subdir, wnScript.name)) os.unlink(this_script) gzipFile(scriptArchive, scriptArchive+'.gz', True) app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def generateJobScripts(app, appendJobScripts): """ Construct a DIRAC scripts which must be unique to each job to have unique checksum. This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput Args: app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running) """ job = app.getJobObject() if not job.master: rjobs = job.subjobs or [job] else: rjobs = [job] tmp_dir = tempfile.gettempdir() # First create the extraOpts files needed 1 per subjob for this_job in rjobs: logger.debug("RTHandler Making Scripts: %s" % this_job.fqid) this_job.application.constructExtraFiles(this_job) if not job.master and job.subjobs: for sj in rjobs: sj.application.jobScriptArchive = sj.master.application.jobScriptArchive master_job = job.master or job # Now lets get the name of this tar file scriptArchive = os.path.join(master_job.application.jobScriptArchive.localDir, master_job.application.jobScriptArchive.namePattern) if appendJobScripts: # Now lets add the Job scripts to this archive with tarfile.open(scriptArchive, 'a') as tar_file: for this_job in rjobs: this_app = this_job.application wnScript = generateWNScript(prepareCommand(this_app), this_app) this_script = os.path.join(tmp_dir, wnScript.name) wnScript.create(this_script) tar_file.add(this_script, arcname=os.path.join(wnScript.subdir, wnScript.name)) gzipFile(scriptArchive, scriptArchive+'.gz', True) app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def prepare(self, force=False): super(GaudiPython, self).prepare(force) self._check_inputs() share_dir = os.path.join( expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], self.is_prepared.name) fillPackedSandbox( self.script, os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name)) gzipFile( os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name), os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare()
if not os.path.isdir(share_path): os.makedirs(share_path) this_file = gzip.GzipFile(os.path.join(share_path, 'gaudi-env.py.gz'), 'wb') this_file.write('gaudi_env = %s' % str(self.getenv(True))) this_file.close() self._parse_options() #try: # self._parse_options() #except Exception, err: # logger.debug("_parse_options Error:\n%s" % str(err)) # self.unprepare() # raise err gzipFile(os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name), os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() # create hash def _check_inputs(self): """Checks the validity of some of user's entries for Gaudi schema""" # Warn user that no optsfiles given if len(self.optsfile) == 0: logger.warning("The 'optsfile' is not set. I hope this is OK!") else: # Check for non-exting optsfiles defined nonexistentOptFiles = []
this_file = gzip.GzipFile(os.path.join(share_path, 'gaudi-env.py.gz'), 'wb') this_file.write('gaudi_env = %s' % str(self.getenv(True))) this_file.close() self._parse_options() #try: # self._parse_options() #except Exception, err: # logger.debug("_parse_options Error:\n%s" % str(err)) # self.unprepare() # raise err gzipFile( os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name), os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True) # add the newly created shared directory into the metadata system if # the app is associated with a persisted object self.checkPreparedHasParent(self) self.post_prepare() # create hash def _check_inputs(self): """Checks the validity of some of user's entries for Gaudi schema""" # Warn user that no optsfiles given if len(self.optsfile) == 0: logger.warning("The 'optsfile' is not set. I hope this is OK!") else: # Check for non-exting optsfiles defined