def prepare(self, force=False):
super(Ostap, self).prepare(force)
self._check_inputs()
share_dir = os.path.join (
expandfilename ( getConfig('Configuration')['gangadir'] ) ,
'shared' ,
getConfig('Configuration')['user'] ,
self.is_prepared.name )
input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name )
input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name )
fillPackedSandbox ( self.scripts , input_sandbox_tar )
gzipFile ( input_sandbox_tar , input_sandbox_tgz , True )
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def prepare(self, force=False):
super(BenderScript, self).prepare(force)
self._check_inputs()
share_dir = os.path.join (
expandfilename ( getConfig('Configuration')['gangadir'] ) ,
'shared' ,
getConfig('Configuration')['user'] ,
self.is_prepared.name )
input_sandbox_tar = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name )
input_sandbox_tgz = os.path.join ( share_dir , 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name )
fillPackedSandbox ( self.scripts + self.imports , input_sandbox_tar )
gzipFile ( input_sandbox_tar , input_sandbox_tgz , True )
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
logger.debug("Finished Preparing Application in %s" % share_dir)
def generateJobScripts(app, appendJobScripts):
"""
Construct a DIRAC scripts which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running)
"""
job = app.getJobObject()
if not job.master:
rjobs = job.subjobs or [job]
else:
rjobs = [job]
tmp_dir = tempfile.gettempdir()
# First create the extraOpts files needed 1 per subjob
for this_job in rjobs:
logger.debug("RTHandler Making Scripts: %s" % this_job.fqid)
this_job.application.constructExtraFiles(this_job)
if not job.master and job.subjobs:
for sj in rjobs:
sj.application.jobScriptArchive = sj.master.application.jobScriptArchive
master_job = job.master or job
# Now lets get the name of this tar file
scriptArchive = os.path.join(
master_job.application.jobScriptArchive.localDir,
master_job.application.jobScriptArchive.namePattern)
if appendJobScripts:
# Now lets add the Job scripts to this archive and potentially the extra options to generate the summary.xml
with tarfile.open(scriptArchive, 'a') as tar_file:
if app.getMetadata:
summaryScript = "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"
summaryPath = os.path.join(job.getInputWorkspace().getPath(),
'summary.py')
summaryFile = FileBuffer(summaryPath, summaryScript)
summaryFile.create()
tar_file.add(summaryPath, arcname='summary.py')
for this_job in rjobs:
this_app = this_job.application
wnScript = generateWNScript(prepareCommand(this_app), this_app)
this_script = os.path.join(tmp_dir, wnScript.name)
wnScript.create(this_script)
tar_file.add(this_script,
arcname=os.path.join(wnScript.subdir,
wnScript.name))
os.unlink(this_script)
gzipFile(scriptArchive, scriptArchive + '.gz', True)
app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def prepare(self, force=False):
super(GaudiPython, self).prepare(force)
self._check_inputs()
share_dir = os.path.join(expandfilename(getConfig('Configuration')['gangadir']), 'shared', getConfig('Configuration')['user'], self.is_prepared.name)
fillPackedSandbox(self.script, os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name))
gzipFile(os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name), True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
def generateJobScripts(app, appendJobScripts):
"""
Construct a DIRAC scripts which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running)
"""
job = app.getJobObject()
if not job.master:
rjobs = job.subjobs or [job]
else:
rjobs = [job]
tmp_dir = tempfile.gettempdir()
# First create the extraOpts files needed 1 per subjob
for this_job in rjobs:
logger.debug("RTHandler Making Scripts: %s" % this_job.fqid)
this_job.application.constructExtraFiles(this_job)
if not job.master and job.subjobs:
for sj in rjobs:
sj.application.jobScriptArchive = sj.master.application.jobScriptArchive
master_job = job.master or job
# Now lets get the name of this tar file
scriptArchive = os.path.join(master_job.application.jobScriptArchive.localDir, master_job.application.jobScriptArchive.namePattern)
if appendJobScripts:
# Now lets add the Job scripts to this archive and potentially the extra options to generate the summary.xml
with tarfile.open(scriptArchive, 'a') as tar_file:
if app.getMetadata:
summaryScript = "\nfrom Gaudi.Configuration import *\nfrom Configurables import LHCbApp\nLHCbApp().XMLSummary='summary.xml'"
summaryPath = os.path.join(job.getInputWorkspace().getPath(), 'summary.py')
summaryFile = FileBuffer(summaryPath, summaryScript)
summaryFile.create()
tar_file.add(summaryPath, arcname = 'summary.py')
for this_job in rjobs:
this_app = this_job.application
wnScript = generateWNScript(prepareCommand(this_app), this_app)
this_script = os.path.join(tmp_dir, wnScript.name)
wnScript.create(this_script)
tar_file.add(this_script, arcname=os.path.join(wnScript.subdir, wnScript.name))
os.unlink(this_script)
gzipFile(scriptArchive, scriptArchive+'.gz', True)
app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def generateJobScripts(app, appendJobScripts):
"""
Construct a DIRAC scripts which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running)
"""
job = app.getJobObject()
if not job.master:
rjobs = job.subjobs or [job]
else:
rjobs = [job]
tmp_dir = tempfile.gettempdir()
# First create the extraOpts files needed 1 per subjob
for this_job in rjobs:
logger.debug("RTHandler Making Scripts: %s" % this_job.fqid)
this_job.application.constructExtraFiles(this_job)
if not job.master and job.subjobs:
for sj in rjobs:
sj.application.jobScriptArchive = sj.master.application.jobScriptArchive
master_job = job.master or job
# Now lets get the name of this tar file
scriptArchive = os.path.join(master_job.application.jobScriptArchive.localDir, master_job.application.jobScriptArchive.namePattern)
if appendJobScripts:
# Now lets add the Job scripts to this archive
with tarfile.open(scriptArchive, 'a') as tar_file:
for this_job in rjobs:
this_app = this_job.application
wnScript = generateWNScript(prepareCommand(this_app), this_app)
this_script = os.path.join(tmp_dir, wnScript.name)
wnScript.create(this_script)
tar_file.add(this_script, arcname=os.path.join(wnScript.subdir, wnScript.name))
gzipFile(scriptArchive, scriptArchive+'.gz', True)
app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def generateJobScripts(app, appendJobScripts):
"""
Construct a DIRAC scripts which must be unique to each job to have unique checksum.
This generates a unique file, uploads it to DRIAC and then stores the LFN in app.uploadedInput
Args:
app (GaudiExec): This expects a GaudiExec app to be passed so that the constructed
appendJobScripts (bool): Should we add the job scripts to the script archive? (Only makes sense on backends which auto-extact tarballs before running)
"""
job = app.getJobObject()
if not job.master:
rjobs = job.subjobs or [job]
else:
rjobs = [job]
tmp_dir = tempfile.gettempdir()
# First create the extraOpts files needed 1 per subjob
for this_job in rjobs:
logger.debug("RTHandler Making Scripts: %s" % this_job.fqid)
this_job.application.constructExtraFiles(this_job)
if not job.master and job.subjobs:
for sj in rjobs:
sj.application.jobScriptArchive = sj.master.application.jobScriptArchive
master_job = job.master or job
# Now lets get the name of this tar file
scriptArchive = os.path.join(master_job.application.jobScriptArchive.localDir, master_job.application.jobScriptArchive.namePattern)
if appendJobScripts:
# Now lets add the Job scripts to this archive
with tarfile.open(scriptArchive, 'a') as tar_file:
for this_job in rjobs:
this_app = this_job.application
wnScript = generateWNScript(prepareCommand(this_app), this_app)
this_script = os.path.join(tmp_dir, wnScript.name)
wnScript.create(this_script)
tar_file.add(this_script, arcname=os.path.join(wnScript.subdir, wnScript.name))
gzipFile(scriptArchive, scriptArchive+'.gz', True)
app.jobScriptArchive.namePattern = app.jobScriptArchive.namePattern + '.gz'
def prepare(self, force=False):
super(GaudiPython, self).prepare(force)
self._check_inputs()
share_dir = os.path.join(
expandfilename(getConfig('Configuration')['gangadir']), 'shared',
getConfig('Configuration')['user'], self.is_prepared.name)
fillPackedSandbox(
self.script,
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name))
gzipFile(
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name),
True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare()
if not os.path.isdir(share_path):
os.makedirs(share_path)
this_file = gzip.GzipFile(os.path.join(share_path, 'gaudi-env.py.gz'), 'wb')
this_file.write('gaudi_env = %s' % str(self.getenv(True)))
this_file.close()
self._parse_options()
#try:
# self._parse_options()
#except Exception, err:
# logger.debug("_parse_options Error:\n%s" % str(err))
# self.unprepare()
# raise err
gzipFile(os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox', '_input_sandbox_%s.tgz' % self.is_prepared.name),
True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare() # create hash
def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
nonexistentOptFiles = []
this_file = gzip.GzipFile(os.path.join(share_path, 'gaudi-env.py.gz'),
'wb')
this_file.write('gaudi_env = %s' % str(self.getenv(True)))
this_file.close()
self._parse_options()
#try:
# self._parse_options()
#except Exception, err:
# logger.debug("_parse_options Error:\n%s" % str(err))
# self.unprepare()
# raise err
gzipFile(
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name),
os.path.join(share_dir, 'inputsandbox',
'_input_sandbox_%s.tgz' % self.is_prepared.name),
True)
# add the newly created shared directory into the metadata system if
# the app is associated with a persisted object
self.checkPreparedHasParent(self)
self.post_prepare() # create hash
def _check_inputs(self):
"""Checks the validity of some of user's entries for Gaudi schema"""
# Warn user that no optsfiles given
if len(self.optsfile) == 0:
logger.warning("The 'optsfile' is not set. I hope this is OK!")
else:
# Check for non-exting optsfiles defined
