def __get_app_exitcode__(outputdir):
import Ganga.Core.Sandbox as Sandbox
Sandbox.getPackedOutputSandbox(outputdir, outputdir)
# check the application exit code
app_exitcode = -1
runtime_log = os.path.join(outputdir, '__jobscript__.log')
pat = re.compile(r'.*exit code (\d+).')
if not os.path.exists(runtime_log):
logger.warning('job runtime log not found: %s' % runtime_log)
return False, 'job runtime log not found: %s' % runtime_log
for line in open(runtime_log, 'r'):
mat = pat.match(line)
if mat:
app_exitcode = eval(mat.groups()[0])
break
# returns False if the exit code of the real executable is not zero
# the job status of GANGA will be changed to 'failed' if the return
# value is False
if app_exitcode != 0:
logger.debug(
'job\'s executable returns non-zero exit code: %d' % app_exitcode)
return False, app_exitcode
else:
return True, 0
def __get_app_exitcode__(outputdir):
import Ganga.Core.Sandbox as Sandbox
Sandbox.getPackedOutputSandbox(outputdir, outputdir)
# check the application exit code
app_exitcode = -1
runtime_log = os.path.join(outputdir, '__jobscript__.log')
pat = re.compile(r'.*exit code (\d+).')
if not os.path.exists(runtime_log):
logger.warning('job runtime log not found: %s' % runtime_log)
return False, 'job runtime log not found: %s' % runtime_log
for line in open(runtime_log, 'r'):
mat = pat.match(line)
if mat:
app_exitcode = eval(mat.groups()[0])
break
# returns False if the exit code of the real executable is not zero
# the job status of GANGA will be changed to 'failed' if the return
# value is False
if app_exitcode != 0:
logger.debug('job\'s executable returns non-zero exit code: %d' %
app_exitcode)
return False, app_exitcode
else:
return True, 0
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###', repr(getName(job.application)))
script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', str(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer('__jobscript__', script), executable=1)
return scriptPath
def prestagesandbox(self, jobconfig):
job = self.getJobObject()
mon = job.getMonitoringService()
inw = job.getInputWorkspace()
sandbox_files = jobconfig.getSandboxFiles()
logger.info("pre-staging files for saga job with id: %s", job.id)
## Compression is ENABLED. All input sandbox files are archived
## into an uncompressed tarball which is then transfered to the
## execution host. This speeds up things a lot in many scenarios.
##
if self.enable_compression:
logger.info(" * adding %s user defined files to input sandbox", len(sandbox_files))
import Ganga.Core.Sandbox as Sandbox
compressed_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
)
try:
for f in compressed_input_sandbox:
archive_url = saga.url("file://localhost/" + f)
logger.info(
" * copying %s -> %s ", archive_url.url, self.filesystem_url + "/" + self.workdir_uuid + "/"
)
sf = saga.filesystem.file(archive_url.url)
sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/", saga.filesystem.Overwrite)
except saga.exception as e:
logger.error("exception caught while transfering file: %s", e.get_all_exceptions())
job.updateStatus("failed")
return False
## Compression is DISABLED. All input sandbox files are transfered
## one-by-one to the execution host. This can be very slow, especially
## if SAGA is using the Globus GridFTP adaptor.
else:
logger.info("prestaging %s input sandbox files", len(sandbox_files))
for f in sandbox_files:
try:
source = saga.url(f.name)
if len(source.scheme) == 0:
source.scheme = "file"
if len(source.host) == 0:
source.host = "localhost"
if f.subdir != ".": # create subdirectory & copy
target = saga.url(self.filesystem_url + "/" + self.workdir_uuid + "/" + f.subdir + "/")
sd = saga.filesystem.directory(target, saga.filesystem.Create)
sf = saga.filesystem.file(source)
logger.info(" * copying %s -> %s ", source, target)
sf.copy(target, saga.filesystem.Overwrite) # copy the file
else: # copy to remote work dir
logger.info(
" * copying %s -> %s ", source, self.filesystem_url + "/" + self.workdir_uuid + "/"
)
sf = saga.filesystem.file(source)
sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/", saga.filesystem.Overwrite)
except saga.exception as e:
logger.error("exception caught while transfering file: %s", e.get_all_exceptions())
job.updateStatus("failed")
return False
##
## copy the job script
##
try:
jobscript_path = saga.url("file://localhost/" + inw.getPath() + "/__jobscript__")
if os.path.exists(jobscript_path.path):
logger.info(
" * copying %s -> %s ", jobscript_path.url, self.filesystem_url + "/" + self.workdir_uuid + "/"
)
sf = saga.filesystem.file(jobscript_path.url)
sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/", saga.filesystem.Overwrite)
except saga.exception as e:
logger.error("exception caught while transfering file: %s", e.get_all_exceptions())
job.updateStatus("failed")
return False
return True
def preparejob(self, jobconfig, master_job_sandbox):
'''Prepare the JDL'''
script = self.__jobWrapperTemplate__()
job = self.getJobObject()
inpw = job.getInputWorkspace()
wrapperlog = '__jobscript__.log'
import Ganga.Core.Sandbox as Sandbox
# FIXME: check what happens if 'stdout','stderr' are specified here
script = script.replace(
'###OUTPUTSANDBOX###', repr(jobconfig.outputbox))
script = script.replace(
'###APPLICATION_NAME###', getName(job.application))
script = script.replace(
'###APPLICATIONEXEC###', repr(jobconfig.getExeString()))
script = script.replace(
'###APPLICATIONARGS###', repr(jobconfig.getArguments()))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
script = script.replace(
'###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ' '))
script = script.replace(
'###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ' '))
if jobconfig.env:
script = script.replace(
'###APPLICATIONENVS###', repr(jobconfig.env))
else:
script = script.replace('###APPLICATIONENVS###', repr({}))
script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
import inspect
script = script.replace(
'###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
mon = job.getMonitoringService()
self.monInfo = None
# set the monitoring file by default to the stdout
if isinstance(self.monInfo, dict):
self.monInfo['remotefile'] = 'stdout'
# try to print out the monitoring service information in debug mode
try:
logger.debug('job info of monitoring service: %s' %
str(self.monInfo))
except:
pass
# prepare input/output sandboxes
packed_files = jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
sandbox_files = job.createPackedInputSandbox(packed_files)
# sandbox of child jobs should include master's sandbox
sandbox_files.extend(master_job_sandbox)
# check the input file size and pre-upload larger inputs to the iocache
lfc_host = ''
input_sandbox_uris = []
input_sandbox_names = []
ick = True
max_prestaged_fsize = 0
for f in sandbox_files:
idx = self.__check_and_prestage_inputfile__(f)
if not idx:
logger.error('input sandbox preparation failed: %s' % f)
ick = False
break
else:
if idx['lfc_host']:
lfc_host = idx['lfc_host']
if idx['remote']:
abspath = os.path.abspath(f)
fsize = os.path.getsize(abspath)
if fsize > max_prestaged_fsize:
max_prestaged_fsize = fsize
input_sandbox_uris.append(
idx['remote'][os.path.basename(f)])
input_sandbox_names.append(
os.path.basename(urlparse(f)[2]))
if idx['local']:
input_sandbox_uris += idx['local']
input_sandbox_names.append(os.path.basename(f))
if not ick:
logger.error('stop job submission')
return None
# determin the lcg-cp timeout according to the max_prestaged_fsize
# - using the assumption of 1 MB/sec.
max_prestaged_fsize = 0
lfc_host = ''
transfer_timeout = config['SandboxTransferTimeout']
predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))
if predict_timeout > transfer_timeout:
transfer_timeout = predict_timeout
if transfer_timeout < 60:
transfer_timeout = 60
script = script.replace(
'###TRANSFERTIMEOUT###', '%d' % transfer_timeout)
# update the job wrapper with the inputsandbox list
script = script.replace(
'###INPUTSANDBOX###', repr({'remote': {}, 'local': input_sandbox_names}))
# write out the job wrapper and put job wrapper into job's inputsandbox
scriptPath = inpw.writefile(
FileBuffer('__jobscript_%s__' % job.getFQID('.'), script), executable=1)
input_sandbox = input_sandbox_uris + [scriptPath]
for isb in input_sandbox:
logger.debug('ISB URI: %s' % isb)
# compose output sandbox to include by default the following files:
# - gzipped stdout (transferred only when the JobLogHandler is WMS)
# - gzipped stderr (transferred only when the JobLogHandler is WMS)
# - __jobscript__.log (job wrapper's log)
output_sandbox = [wrapperlog]
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
for outputSandboxPattern in getOutputSandboxPatterns(job):
output_sandbox.append(outputSandboxPattern)
if config['JobLogHandler'] in ['WMS']:
output_sandbox += ['stdout.gz', 'stderr.gz']
if len(jobconfig.outputbox):
output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]
# compose LCG JDL
jdl = {
'VirtualOrganisation': config['VirtualOrganisation'],
'Executable': os.path.basename(scriptPath),
'Environment': {'GANGA_LCG_VO': config['VirtualOrganisation'], 'GANGA_LOG_HANDLER': config['JobLogHandler'], 'LFC_HOST': lfc_host},
'StdOutput': 'stdout',
'StdError': 'stderr',
'InputSandbox': input_sandbox,
'OutputSandbox': output_sandbox,
'OutputSandboxBaseDestURI': 'gsiftp://localhost'
}
jdl['Environment'].update({'GANGA_LCG_CE': self.CE})
jdl['Requirements'] = self.requirements.merge(
jobconfig.requirements).convert()
if self.jobtype.upper() in ['NORMAL', 'MPICH']:
jdl['JobType'] = self.jobtype.upper()
if self.jobtype.upper() == 'MPICH':
#jdl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
jdl['Requirements'].append(
'Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
jdl['NodeNumber'] = self.requirements.nodenumber
else:
logger.warning('JobType "%s" not supported' % self.jobtype)
return
# additional settings from the job
# if jobconfig.env:
# jdl['Environment'].update(jobconfig.env)
jdlText = Grid.expandjdl(jdl)
logger.debug('subjob JDL: %s' % jdlText)
return inpw.writefile(FileBuffer('__jdlfile__', jdlText))
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles()
+
Sandbox.getGangaModulesAsSandboxFiles(
Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice', '-n %d' % self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath = job.getOutputWorkspace().getPath()
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config['location'])
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'LocalHostExec.py')
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, '')
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles, getWNCodeForInputdataListCreation
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###', getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__'], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ''))
script = script.replace('###CREATEINPUTDATALIST###', getWNCodeForInputdataListCreation(job, ''))
script = script.replace('###APPLICATION_NAME###', repr(job.application._name))
script = script.replace('###INPUT_SANDBOX###', repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###', repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###', repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###', repr(outputpatterns))
script = script.replace('###JOBID###', jobidRepr)
script = script.replace('###ENVIRONMENT###', repr(environment))
script = script.replace('###WORKDIR###', repr(workdir))
script = script.replace('###INPUT_DIR###', repr(job.getStringInputDir()))
script = script.replace('###MONITORING_SERVICE###', job.getMonitoringService().getWrapperScriptConstructorText())
self.workdir = workdir
script = script.replace('###GANGADIR###', repr(getConfig('System')['GANGA_PYTHONPATH']))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(
FileBuffer('__jobscript__', script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME Check this isn't a GangaList
outputpatterns = jobconfig.outputbox
environment = jobconfig.env if not jobconfig.env is None else {}
import inspect
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'BatchScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
text = FileUtils.loadScript(script_location, '')
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
jobidRepr = repr(self.getJobObject().getFQID('.'))
replace_dict = {
'###OUTPUTSANDBOXPOSTPROCESSING###' : getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
'###OUTPUTUPLOADSPOSTPROCESSING###' : getWNCodeForOutputPostprocessing(job, ''),
'###DOWNLOADINPUTFILES###' : getWNCodeForDownloadingInputFiles(job, ''),
'###INLINEMODULES###' : inspect.getsource(Sandbox.WNSandbox),
'###INLINEHOSTNAMEFUNCTION###' : inspect.getsource(Utility.util.hostname),
'###APPSCRIPTPATH###' : repr(appscriptpath),
#'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
'###INPUT_SANDBOX###' : repr(subjob_input_sandbox + master_input_sandbox),
'###SHAREDOUTPUTPATH###' : repr(sharedoutputpath),
'###OUTPUTPATTERNS###' : repr(outputpatterns),
'###JOBID###' : jobidRepr,
'###ENVIRONMENT###' : repr(environment),
'###PREEXECUTE###' : self.config['preexecute'],
'###POSTEXECUTE###' : self.config['postexecute'],
'###JOBIDNAME###' : self.config['jobid_name'],
'###QUEUENAME###' : self.config['queue_name'],
'###HEARTBEATFREQUENCE###' : self.config['heartbeat_frequency'],
'###INPUT_DIR###' : repr(job.getStringInputDir()),
'###GANGADIR###' : repr(getConfig('System')['GANGA_PYTHONPATH'])
}
for k, v in replace_dict.iteritems():
text = text.replace(str(k), str(v))
logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
logger.debug('master input sandbox %s ', master_input_sandbox)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__', text), executable=1)
def preparejob(self, jobconfig, master_job_sandbox):
'''Prepare the JDL'''
script = self.__jobWrapperTemplate__()
job = self.getJobObject()
inpw = job.getInputWorkspace()
wrapperlog = '__jobscript__.log'
import Ganga.Core.Sandbox as Sandbox
# FIXME: check what happens if 'stdout','stderr' are specified here
script = script.replace(
'###OUTPUTSANDBOX###', repr(jobconfig.outputbox))
script = script.replace(
'###APPLICATION_NAME###', getName(job.application))
script = script.replace(
'###APPLICATIONEXEC###', repr(jobconfig.getExeString()))
script = script.replace(
'###APPLICATIONARGS###', repr(jobconfig.getArguments()))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
script = script.replace(
'###OUTPUTUPLOADSPOSTPROCESSING###', getWNCodeForOutputPostprocessing(job, ' '))
script = script.replace(
'###DOWNLOADINPUTFILES###', getWNCodeForDownloadingInputFiles(job, ' '))
if jobconfig.env:
script = script.replace(
'###APPLICATIONENVS###', repr(jobconfig.env))
else:
script = script.replace('###APPLICATIONENVS###', repr({}))
script = script.replace('###WRAPPERLOG###', repr(wrapperlog))
import inspect
script = script.replace(
'###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
mon = job.getMonitoringService()
self.monInfo = None
# set the monitoring file by default to the stdout
if isinstance(self.monInfo, dict):
self.monInfo['remotefile'] = 'stdout'
# try to print out the monitoring service information in debug mode
try:
logger.debug('job info of monitoring service: %s' %
str(self.monInfo))
except:
pass
# prepare input/output sandboxes
packed_files = jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
sandbox_files = job.createPackedInputSandbox(packed_files)
# sandbox of child jobs should include master's sandbox
sandbox_files.extend(master_job_sandbox)
# check the input file size and pre-upload larger inputs to the iocache
lfc_host = ''
input_sandbox_uris = []
input_sandbox_names = []
ick = True
max_prestaged_fsize = 0
for f in sandbox_files:
idx = self.__check_and_prestage_inputfile__(f)
if not idx:
logger.error('input sandbox preparation failed: %s' % f)
ick = False
break
else:
if idx['lfc_host']:
lfc_host = idx['lfc_host']
if idx['remote']:
abspath = os.path.abspath(f)
fsize = os.path.getsize(abspath)
if fsize > max_prestaged_fsize:
max_prestaged_fsize = fsize
input_sandbox_uris.append(
idx['remote'][os.path.basename(f)])
input_sandbox_names.append(
os.path.basename(urlparse(f)[2]))
if idx['local']:
input_sandbox_uris += idx['local']
input_sandbox_names.append(os.path.basename(f))
if not ick:
logger.error('stop job submission')
return None
# determin the lcg-cp timeout according to the max_prestaged_fsize
# - using the assumption of 1 MB/sec.
max_prestaged_fsize = 0
lfc_host = ''
transfer_timeout = config['SandboxTransferTimeout']
predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))
if predict_timeout > transfer_timeout:
transfer_timeout = predict_timeout
if transfer_timeout < 60:
transfer_timeout = 60
script = script.replace(
'###TRANSFERTIMEOUT###', '%d' % transfer_timeout)
# update the job wrapper with the inputsandbox list
script = script.replace(
'###INPUTSANDBOX###', repr({'remote': {}, 'local': input_sandbox_names}))
# write out the job wrapper and put job wrapper into job's inputsandbox
scriptPath = inpw.writefile(
FileBuffer('__jobscript_%s__' % job.getFQID('.'), script), executable=1)
input_sandbox = input_sandbox_uris + [scriptPath]
for isb in input_sandbox:
logger.debug('ISB URI: %s' % isb)
# compose output sandbox to include by default the following files:
# - gzipped stdout (transferred only when the JobLogHandler is WMS)
# - gzipped stderr (transferred only when the JobLogHandler is WMS)
# - __jobscript__.log (job wrapper's log)
output_sandbox = [wrapperlog]
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
for outputSandboxPattern in getOutputSandboxPatterns(job):
output_sandbox.append(outputSandboxPattern)
if config['JobLogHandler'] in ['WMS']:
output_sandbox += ['stdout.gz', 'stderr.gz']
if len(jobconfig.outputbox):
output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]
# compose ARC XRSL
xrsl = {
#'VirtualOrganisation' : config['VirtualOrganisation'],
'executable': os.path.basename(scriptPath),
'environment': {'GANGA_LCG_VO': config['VirtualOrganisation'], 'GANGA_LOG_HANDLER': config['JobLogHandler'], 'LFC_HOST': lfc_host},
#'stdout' : 'stdout',
#'stderr' : 'stderr',
'inputFiles': input_sandbox,
'outputFiles': output_sandbox,
#'OutputSandboxBaseDestURI': 'gsiftp://localhost'
}
xrsl['environment'].update({'GANGA_LCG_CE': self.CE})
#xrsl['Requirements'] = self.requirements.merge(jobconfig.requirements).convert()
# if self.jobtype.upper() in ['NORMAL','MPICH']:
#xrsl['JobType'] = self.jobtype.upper()
# if self.jobtype.upper() == 'MPICH':
#xrsl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
# xrsl['Requirements'].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
#xrsl['NodeNumber'] = self.requirements.nodenumber
# else:
# logger.warning('JobType "%s" not supported' % self.jobtype)
# return
# additional settings from the job
if jobconfig.env:
xrsl['environment'].update(jobconfig.env)
xrslText = Grid.expandxrsl(xrsl)
# append any additional requirements from the requirements object
xrslText += '\n'.join(self.requirements.other)
logger.debug('subjob XRSL: %s' % xrslText)
return inpw.writefile(FileBuffer('__xrslfile__', xrslText))
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
# print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles() + Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
)
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
if self.nice:
appscriptpath = ["nice", "-n %d" % self.nice] + appscriptpath
if self.nice < 0:
logger.warning("increasing process priority is often not allowed, your job may fail due to this")
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME DON'T just use the blind list here, request the list of files to be in the output from a method.
outputpatterns = jobconfig.outputbox
environment = dict() if jobconfig.env is None else jobconfig.env
import tempfile
workdir = tempfile.mkdtemp(dir=config["location"])
import inspect
script_location = os.path.join(
os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))), "LocalHostExec.py"
)
from Ganga.GPIDev.Lib.File import FileUtils
script = FileUtils.loadScript(script_location, "")
script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import (
getWNCodeForOutputSandbox,
getWNCodeForOutputPostprocessing,
getWNCodeForDownloadingInputFiles,
getWNCodeForInputdataListCreation,
)
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID("."))
script = script.replace(
"###OUTPUTSANDBOXPOSTPROCESSING###",
getWNCodeForOutputSandbox(job, ["stdout", "stderr", "__syslog__"], jobidRepr),
)
script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))
script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, ""))
script = script.replace("###CREATEINPUTDATALIST###", getWNCodeForInputdataListCreation(job, ""))
script = script.replace("###APPLICATION_NAME###", repr(getName(job.application)))
script = script.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
script = script.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))
script = script.replace("###APPSCRIPTPATH###", repr(appscriptpath))
script = script.replace("###OUTPUTPATTERNS###", str(outputpatterns))
script = script.replace("###JOBID###", jobidRepr)
script = script.replace("###ENVIRONMENT###", repr(environment))
script = script.replace("###WORKDIR###", repr(workdir))
script = script.replace("###INPUT_DIR###", repr(job.getStringInputDir()))
self.workdir = workdir
script = script.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))
wrkspace = job.getInputWorkspace()
scriptPath = wrkspace.writefile(FileBuffer("__jobscript__", script), executable=1)
return scriptPath
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
)
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
outputpatterns = jobconfig.outputbox
environment = jobconfig.env
text = """#!/usr/bin/env python
import shutil
import os
import time
import popen2
import glob
############################################################################################
###INLINEMODULES###
###INLINEHOSTNAMEFUNCTION###
############################################################################################
input_sandbox = ###INPUT_SANDBOX###
sharedoutputpath = ###SHAREDOUTPUTPATH###
outputpatterns = ###OUTPUTPATTERNS###
appscriptpath = ###APPSCRIPTPATH###
environment = ###ENVIRONMENT###
# jobid is a string
jobid = ###JOBID###
###PREEXECUTE###
def flush_file(f):
f.flush()
os.fsync(f.fileno()) #this forces a global flush (cache synchronization on AFS)
def open_file(fname):
try:
filehandle=file(fname,'w')
except IOError,x:
print 'ERROR: not able to write a status file: ', fname
print 'ERROR: ',x
raise
return filehandle
statusfilename = os.path.join(sharedoutputpath,'__jobstatus__')
heartbeatfilename = os.path.join(sharedoutputpath,'__heartbeat__')
statusfile=open_file(statusfilename)
heartbeatfile=open_file(heartbeatfilename)
line='START: '+ time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
try:
line+='PID: ' + os.getenv('###JOBIDNAME###') + os.linesep
line+='QUEUE: ' + os.getenv('###QUEUENAME###') + os.linesep
line+='ACTUALCE: ' + hostname() + os.linesep
except:
pass
statusfile.writelines(line)
flush_file(statusfile)
try:
import tarfile
except ImportError,x:
sys.path.insert(0,###TARFILE_PYTHONPATH###)
import tarfile
# -- WARNING: get the input files including the python modules BEFORE sys.path.insert()
# -- SINCE PYTHON 2.6 THERE WAS A SUBTLE CHANGE OF SEMANTICS IN THIS AREA
for f in input_sandbox:
getPackedInputSandbox(f)
# -- END OF MOVED CODE BLOCK
import sys
sys.path.insert(0, ###GANGADIR###)
sys.path.insert(0,os.path.join(os.getcwd(),PYTHON_DIR))
try:
import subprocess
except ImportError,x:
sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
import subprocess
for key,value in environment.iteritems():
os.environ[key] = value
sysout2 = os.dup(sys.stdout.fileno())
syserr2 = os.dup(sys.stderr.fileno())
print >>sys.stdout,"--- GANGA APPLICATION OUTPUT BEGIN ---"
print >>sys.stderr,"--- GANGA APPLICATION ERROR BEGIN ---"
flush_file(sys.stdout)
flush_file(sys.stderr)
sys.stdout=file('./__syslog__','w')
sys.stderr=sys.stdout
###MONITORING_SERVICE###
monitor = createMonitoringObject()
monitor.start()
result = 255
try:
child = subprocess.Popen(appscriptpath, shell=False, stdout=sysout2, stderr=syserr2)
while 1:
result = child.poll()
if result is not None:
break
monitor.progress()
heartbeatfile.write('.')
flush_file(heartbeatfile)
time.sleep(###HEARTBEATFREQUENCE###)
except Exception,x:
print 'ERROR: %s'%str(x)
monitor.progress()
flush_file(sys.stdout)
flush_file(sys.stderr)
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
print >>sys.stdout,"--- GANGA APPLICATION OUTPUT END ---"
monitor.stop(result)
try:
filefilter
except:
filefilter = None
from Ganga.Utility.files import multi_glob, recursive_copy
createOutputSandbox(outputpatterns,filefilter,sharedoutputpath)
def printError(message):
print >>sys.stderr, message
def printInfo(message):
print >>sys.stdout, message
###OUTPUTUPLOADSPOSTPROCESSING###
print >>sys.stderr,"--- GANGA APPLICATION ERROR END ---"
###OUTPUTSANDBOXPOSTPROCESSING###
###POSTEXECUTE###
line='EXITCODE: ' + repr(result) + os.linesep
line+='STOP: '+time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
statusfile.close()
heartbeatfile.close()
os.unlink(heartbeatfilename)
sys.exit(result)
"""
import inspect
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing
jobidRepr = repr(self.getJobObject().getFQID("."))
text = text.replace(
"###OUTPUTSANDBOXPOSTPROCESSING###",
getWNCodeForOutputSandbox(
job, ["__syslog__", getConfig("Output")["PostProcessLocationsFileName"]], jobidRepr
),
)
text = text.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, ""))
text = text.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))
text = text.replace("###INLINEHOSTNAMEFUNCTION###", inspect.getsource(Utility.util.hostname))
text = text.replace("###APPSCRIPTPATH###", repr(appscriptpath))
# text = text.replace('###SHAREDINPUTPATH###',repr(sharedinputpath))
logger.debug("subjob input sandbox %s ", subjob_input_sandbox)
logger.debug("master input sandbox %s ", master_input_sandbox)
text = text.replace("###INPUT_SANDBOX###", repr(subjob_input_sandbox + master_input_sandbox))
text = text.replace("###SHAREDOUTPUTPATH###", repr(sharedoutputpath))
text = text.replace("###OUTPUTPATTERNS###", repr(outputpatterns))
text = text.replace("###JOBID###", jobidRepr)
text = text.replace("###ENVIRONMENT###", repr(environment))
text = text.replace("###PREEXECUTE###", self.config["preexecute"])
text = text.replace("###POSTEXECUTE###", self.config["postexecute"])
text = text.replace("###JOBIDNAME###", self.config["jobid_name"])
text = text.replace("###QUEUENAME###", self.config["queue_name"])
text = text.replace("###HEARTBEATFREQUENCE###", self.config["heartbeat_frequency"])
text = text.replace("###INPUT_DIR###", repr(job.getStringInputDir()))
text = text.replace("###MONITORING_SERVICE###", job.getMonitoringService().getWrapperScriptConstructorText())
text = text.replace("###GANGADIR###", repr(getConfig("System")["GANGA_PYTHONPATH"]))
import Ganga.PACKAGE
text = text.replace(
"###SUBPROCESS_PYTHONPATH###",
repr(Ganga.PACKAGE.setup.getPackagePath2("subprocess", "syspath", force=True)),
)
text = text.replace(
"###TARFILE_PYTHONPATH###", repr(Ganga.PACKAGE.setup.getPackagePath2("tarfile", "syspath", force=True))
)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer("__jobscript__", text), executable=1)
def prestagesandbox(self, jobconfig):
job = self.getJobObject()
mon = job.getMonitoringService()
inw = job.getInputWorkspace()
sandbox_files = jobconfig.getSandboxFiles()
logger.info("pre-staging files for saga job with id: %s", job.id)
## Compression is ENABLED. All input sandbox files are archived
## into an uncompressed tarball which is then transfered to the
## execution host. This speeds up things a lot in many scenarios.
##
if self.enable_compression:
logger.info(" * adding %s user defined files to input sandbox",
len(sandbox_files))
import Ganga.Core.Sandbox as Sandbox
compressed_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles() +
Sandbox.getGangaModulesAsSandboxFiles(
Sandbox.getDefaultModules()))
try:
for f in compressed_input_sandbox:
archive_url = saga.url("file://localhost/" + f)
logger.info(
" * copying %s -> %s ", archive_url.url,
self.filesystem_url + "/" + self.workdir_uuid + "/")
sf = saga.filesystem.file(archive_url.url)
sf.copy(
self.filesystem_url + "/" + self.workdir_uuid + "/",
saga.filesystem.Overwrite)
except saga.exception as e:
logger.error('exception caught while transfering file: %s',
e.get_all_exceptions())
job.updateStatus("failed")
return False
## Compression is DISABLED. All input sandbox files are transfered
## one-by-one to the execution host. This can be very slow, especially
## if SAGA is using the Globus GridFTP adaptor.
else:
logger.info("prestaging %s input sandbox files",
len(sandbox_files))
for f in sandbox_files:
try:
source = saga.url(f.name)
if (len(source.scheme) == 0):
source.scheme = "file"
if (len(source.host) == 0):
source.host = "localhost"
if (f.subdir != '.'): # create subdirectory & copy
target = saga.url(self.filesystem_url + "/" +
self.workdir_uuid + "/" + f.subdir +
"/")
sd = saga.filesystem.directory(target,
saga.filesystem.Create)
sf = saga.filesystem.file(source)
logger.info(" * copying %s -> %s ", source, target)
sf.copy(target,
saga.filesystem.Overwrite) # copy the file
else: # copy to remote work dir
logger.info(
" * copying %s -> %s ", source,
self.filesystem_url + "/" + self.workdir_uuid +
"/")
sf = saga.filesystem.file(source)
sf.copy(
self.filesystem_url + "/" + self.workdir_uuid +
"/", saga.filesystem.Overwrite)
except saga.exception as e:
logger.error('exception caught while transfering file: %s',
e.get_all_exceptions())
job.updateStatus("failed")
return False
##
## copy the job script
##
try:
jobscript_path = saga.url("file://localhost/" + inw.getPath() +
"/__jobscript__")
if os.path.exists(jobscript_path.path):
logger.info(
" * copying %s -> %s ", jobscript_path.url,
self.filesystem_url + "/" + self.workdir_uuid + "/")
sf = saga.filesystem.file(jobscript_path.url)
sf.copy(self.filesystem_url + "/" + self.workdir_uuid + "/",
saga.filesystem.Overwrite)
except saga.exception as e:
logger.error('exception caught while transfering file: %s',
e.get_all_exceptions())
job.updateStatus("failed")
return False
return True
class SAGASubmitter(Submitter):
def submit_worker(self,j):
import Ganga.GPI
"""This function uses Ganga.GPI to submit the job created by the submitter script."""
j.application = Ganga.GPI.Executable()
wf = tempfile.NamedTemporaryFile(mode="w",prefix='diane_worker_wrapper-')
# if diane dir is not explicitly given, default to
# the master wrapper
if(self.options.dianedir == ''):
wf.write(self.wrapper)
# if it is given, use a modified wrapper script
else:
sh_run_wrapper = '''#!/bin/sh
$('''+self.options.dianedir+'''/bin/diane-env)
diane-worker-start $*
'''
wf.write(sh_run_wrapper)
wf.flush()
uuid_path = tempfile.mkdtemp()
uuid = open(os.path.join(uuid_path, 'ganga_job_uuid'), 'w')
uuid.write(str(j.info.uuid))
uuid.flush()
uuid.close()
diane.util.chmod_executable(wf.name)
j.application.exe = "/bin/sh" #Ganga.GPI.File(wf.name)
j.application.args=[Ganga.GPI.File(wf.name)]
j.application.args+=['--ior-file=MasterOID']
if self.enable_GSI:
j.application.args+=['--enable-GSI']
j.application.args+=['--omniorb-config-file=%s'%os.path.basename(self.config_file.name)]
# Point the worker agent to the working directory
workdir = saga.url(j.backend.filesystem_url).path+"/"
j.application.args += ['--workdir=%s'%workdir]
from Ganga.Utility.util import hostname
ganga_config = Ganga.GPI.config
# works only for local repository type
j.application.env = {'GANGA_JOB_ID': "%s@%s:%s:%d"%(ganga_config.Configuration.user,ganga_config.System.GANGA_HOSTNAME, ganga_config.Configuration.gangadir, j.id),
'GANGA_JOB_UUID': str(j.info.uuid)}
j.inputsandbox = j.inputsandbox + [Ganga.GPI.File(self.master_oid_filename),
Ganga.GPI.File(self.config_file.name),
Ganga.GPI.File(uuid.name)]
j.name = "SAGA/DIANEWorkerAgent"
if self.options.ENABLE_DOCTOR:
j.outputsandbox = j.outputsandbox + ['*DUMMY_PLATFORM_STRING*']
#try:
self.runfile.call('worker_submit',j)
#except DianeException:
# pass
###############
### EMULATION OF WRAPPER SCRIPT
ganga_wrapper = """
try:
import subprocess
except ImportError,x:
sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
import subprocess
appscriptpath = ###APPSCRIPTPATH###
###MONITORING_SERVICE###
monitor = createMonitoringObject()
monitor.start()
import subprocess
try:
child = subprocess.Popen(appscriptpath, shell=False)
except OSError,x:
print 'EXITCODE: %d'%-9999
print 'PROBLEM STARTING THE APPLICATION SCRIPT: %s %s'%(appscriptpath,str(x))
sys.exit()
result = -1
try:
while 1:
result = child.poll()
if result is not None:
break
monitor.progress()
time.sleep(0.3)
finally:
monitor.progress()
monitor.stop(result)
"""
job = j._impl
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
import Ganga.PACKAGE
j.inputsandbox += Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
j.inputsandbox += Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))
ganga_wrapper = ganga_wrapper.replace('###APPSCRIPTPATH###',repr([j.application.exe,wf.name]+j.application.args[1:])) #HACK SPECIFIC TO THIS SCRIPT
#ganga_wrapper = ganga_wrapper.replace('###APPSCRIPTPATH###',repr([j.application.exe]+j.args)) # NEED TO CONVERT ALL ARGS TO STRINGS (SMARTLY)
ganga_wrapper = ganga_wrapper.replace('###JOBID###',repr(job.getFQID('.')))
ganga_wrapper = ganga_wrapper.replace('###MONITORING_SERVICE###',job.getMonitoringService().getWrapperScriptConstructorText())
import Ganga.PACKAGE
ganga_wrapper = ganga_wrapper.replace('###SUBPROCESS_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('subprocess','syspath',force=True)))
gwf = tempfile.NamedTemporaryFile(mode="w",prefix='ganga-emulated-wrapper-')
gwf.write(ganga_wrapper)
gwf.close()
# replace the job args
j.application.exe='/usr/bin/python'
j.application.args=[Ganga.GPI.File(gwf.name)]
j.submit()
def preparejob(self, jobconfig, master_input_sandbox):
job = self.getJobObject()
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(
jobconfig.getSandboxFiles() +
Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules()))
appscriptpath = [jobconfig.getExeString()] + jobconfig.getArgStrings()
sharedoutputpath = job.getOutputWorkspace().getPath()
## FIXME Check this isn't a GangaList
outputpatterns = jobconfig.outputbox
environment = jobconfig.env if not jobconfig.env is None else {}
import inspect
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'BatchScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
text = FileUtils.loadScript(script_location, '')
import Ganga.Core.Sandbox as Sandbox
import Ganga.Utility as Utility
from Ganga.Utility.Config import getConfig
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing, getWNCodeForDownloadingInputFiles
jobidRepr = repr(self.getJobObject().getFQID('.'))
replace_dict = {
'###OUTPUTSANDBOXPOSTPROCESSING###':
getWNCodeForOutputSandbox(job, ['__syslog__'], jobidRepr),
'###OUTPUTUPLOADSPOSTPROCESSING###':
getWNCodeForOutputPostprocessing(job, ''),
'###DOWNLOADINPUTFILES###':
getWNCodeForDownloadingInputFiles(job, ''),
'###INLINEMODULES###':
inspect.getsource(Sandbox.WNSandbox),
'###INLINEHOSTNAMEFUNCTION###':
inspect.getsource(Utility.util.hostname),
'###APPSCRIPTPATH###':
repr(appscriptpath),
#'###SHAREDINPUTPATH###' : repr(sharedinputpath)),
'###INPUT_SANDBOX###':
repr(subjob_input_sandbox + master_input_sandbox),
'###SHAREDOUTPUTPATH###':
repr(sharedoutputpath),
'###OUTPUTPATTERNS###':
repr(outputpatterns),
'###JOBID###':
jobidRepr,
'###ENVIRONMENT###':
repr(environment),
'###PREEXECUTE###':
self.config['preexecute'],
'###POSTEXECUTE###':
self.config['postexecute'],
'###JOBIDNAME###':
self.config['jobid_name'],
'###QUEUENAME###':
self.config['queue_name'],
'###HEARTBEATFREQUENCE###':
self.config['heartbeat_frequency'],
'###INPUT_DIR###':
repr(job.getStringInputDir()),
'###GANGADIR###':
repr(getConfig('System')['GANGA_PYTHONPATH'])
}
for k, v in replace_dict.iteritems():
text = text.replace(str(k), str(v))
logger.debug('subjob input sandbox %s ', subjob_input_sandbox)
logger.debug('master input sandbox %s ', master_input_sandbox)
from Ganga.GPIDev.Lib.File import FileBuffer
return job.getInputWorkspace().writefile(FileBuffer(
'__jobscript__', text),
executable=1)
def preparejob(self, jobconfig, master_job_sandbox):
"""Prepare the JDL"""
script = self.__jobWrapperTemplate__()
job = self.getJobObject()
inpw = job.getInputWorkspace()
wrapperlog = "__jobscript__.log"
import Ganga.Core.Sandbox as Sandbox
# FIXME: check what happens if 'stdout','stderr' are specified here
script = script.replace("###OUTPUTSANDBOX###", repr(jobconfig.outputbox))
script = script.replace("###APPLICATION_NAME###", job.application._name)
script = script.replace("###APPLICATIONEXEC###", repr(jobconfig.getExeString()))
script = script.replace("###APPLICATIONARGS###", repr(jobconfig.getArguments()))
from Ganga.GPIDev.Lib.File.OutputFileManager import (
getWNCodeForOutputPostprocessing,
getWNCodeForDownloadingInputFiles,
)
script = script.replace("###OUTPUTUPLOADSPOSTPROCESSING###", getWNCodeForOutputPostprocessing(job, " "))
script = script.replace("###DOWNLOADINPUTFILES###", getWNCodeForDownloadingInputFiles(job, " "))
if jobconfig.env:
script = script.replace("###APPLICATIONENVS###", repr(jobconfig.env))
else:
script = script.replace("###APPLICATIONENVS###", repr({}))
script = script.replace("###WRAPPERLOG###", repr(wrapperlog))
import inspect
script = script.replace("###INLINEMODULES###", inspect.getsource(Sandbox.WNSandbox))
mon = job.getMonitoringService()
self.monInfo = None
# set the monitoring file by default to the stdout
if isinstance(self.monInfo, dict):
self.monInfo["remotefile"] = "stdout"
# try to print out the monitoring service information in debug mode
try:
logger.debug("job info of monitoring service: %s" % str(self.monInfo))
except:
pass
script = script.replace("###MONITORING_SERVICE###", mon.getWrapperScriptConstructorText())
# prepare input/output sandboxes
packed_files = (
jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules())
)
sandbox_files = job.createPackedInputSandbox(packed_files)
# sandbox of child jobs should include master's sandbox
sandbox_files.extend(master_job_sandbox)
# check the input file size and pre-upload larger inputs to the iocache
lfc_host = ""
input_sandbox_uris = []
input_sandbox_names = []
ick = True
max_prestaged_fsize = 0
for f in sandbox_files:
idx = self.__check_and_prestage_inputfile__(f)
if not idx:
logger.error("input sandbox preparation failed: %s" % f)
ick = False
break
else:
if idx["lfc_host"]:
lfc_host = idx["lfc_host"]
if idx["remote"]:
abspath = os.path.abspath(f)
fsize = os.path.getsize(abspath)
if fsize > max_prestaged_fsize:
max_prestaged_fsize = fsize
input_sandbox_uris.append(idx["remote"][os.path.basename(f)])
input_sandbox_names.append(os.path.basename(urlparse(f)[2]))
if idx["local"]:
input_sandbox_uris += idx["local"]
input_sandbox_names.append(os.path.basename(f))
if not ick:
logger.error("stop job submission")
return None
# determin the lcg-cp timeout according to the max_prestaged_fsize
# - using the assumption of 1 MB/sec.
max_prestaged_fsize = 0
lfc_host = ""
transfer_timeout = config["SandboxTransferTimeout"]
predict_timeout = int(math.ceil(max_prestaged_fsize / 1000000.0))
if predict_timeout > transfer_timeout:
transfer_timeout = predict_timeout
if transfer_timeout < 60:
transfer_timeout = 60
script = script.replace("###TRANSFERTIMEOUT###", "%d" % transfer_timeout)
# update the job wrapper with the inputsandbox list
script = script.replace("###INPUTSANDBOX###", repr({"remote": {}, "local": input_sandbox_names}))
# write out the job wrapper and put job wrapper into job's inputsandbox
scriptPath = inpw.writefile(FileBuffer("__jobscript_%s__" % job.getFQID("."), script), executable=1)
input_sandbox = input_sandbox_uris + [scriptPath]
for isb in input_sandbox:
logger.debug("ISB URI: %s" % isb)
# compose output sandbox to include by default the following files:
# - gzipped stdout (transferred only when the JobLogHandler is WMS)
# - gzipped stderr (transferred only when the JobLogHandler is WMS)
# - __jobscript__.log (job wrapper's log)
output_sandbox = [wrapperlog]
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatterns
for outputSandboxPattern in getOutputSandboxPatterns(job):
output_sandbox.append(outputSandboxPattern)
if config["JobLogHandler"] in ["WMS"]:
output_sandbox += ["stdout.gz", "stderr.gz"]
if len(jobconfig.outputbox):
output_sandbox += [Sandbox.OUTPUT_TARBALL_NAME]
# compose LCG JDL
jdl = {
"VirtualOrganisation": config["VirtualOrganisation"],
"Executable": os.path.basename(scriptPath),
"Environment": {
"GANGA_LCG_VO": config["VirtualOrganisation"],
"GANGA_LOG_HANDLER": config["JobLogHandler"],
"LFC_HOST": lfc_host,
},
"StdOutput": "stdout",
"StdError": "stderr",
"InputSandbox": input_sandbox,
"OutputSandbox": output_sandbox,
"OutputSandboxBaseDestURI": "gsiftp://localhost",
}
jdl["Environment"].update({"GANGA_LCG_CE": self.CE})
jdl["Requirements"] = self.requirements.merge(jobconfig.requirements).convert()
if self.jobtype.upper() in ["NORMAL", "MPICH"]:
jdl["JobType"] = self.jobtype.upper()
if self.jobtype.upper() == "MPICH":
# jdl['Requirements'].append('(other.GlueCEInfoTotalCPUs >= NodeNumber)')
jdl["Requirements"].append('Member("MPICH",other.GlueHostApplicationSoftwareRunTimeEnvironment)')
jdl["NodeNumber"] = self.requirements.nodenumber
else:
logger.warning('JobType "%s" not supported' % self.jobtype)
return
# additional settings from the job
# if jobconfig.env:
# jdl['Environment'].update(jobconfig.env)
jdlText = Grid.expandjdl(jdl)
logger.debug("subjob JDL: %s" % jdlText)
return inpw.writefile(FileBuffer("__jdlfile__", jdlText))
def preparejob(self,jobconfig,master_input_sandbox):
job = self.getJobObject()
#print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))
appscriptpath = [jobconfig.getExeString()]+jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice','-n %d'%self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath=job.getOutputWorkspace().getPath()
outputpatterns=jobconfig.outputbox
environment=jobconfig.env
from Ganga.Utility import tempfile
workdir = tempfile.mkdtemp()
script= """#!/usr/bin/env python
import os,os.path,shutil,tempfile
import sys,time
import glob
import sys
# FIXME: print as DEBUG: to __syslog__ file
#print sys.path
#print os.environ['PATH']
#print sys.version
# bugfix #13314 : make sure that the wrapper (spawned process) is detached from Ganga session
# the process will not receive Control-C signals
# using fork and doing setsid() before exec would probably be a bit
# better (to avoid slim chance that the signal is propagated to this
# process before setsid is reached)
# this is only enabled if the first argument is 'subprocess' in order to enable
# running this script by hand from outside ganga (which is sometimes useful)
if len(sys.argv)>1 and sys.argv[1] == 'subprocess':
os.setsid()
############################################################################################
###INLINEMODULES###
############################################################################################
input_sandbox = ###INPUT_SANDBOX###
sharedoutputpath= ###SHAREDOUTPUTPATH###
outputpatterns = ###OUTPUTPATTERNS###
appscriptpath = ###APPSCRIPTPATH###
environment = ###ENVIRONMENT###
workdir = ###WORKDIR###
statusfilename = os.path.join(sharedoutputpath,'__jobstatus__')
try:
statusfile=file(statusfilename,'w')
except IOError,x:
print 'ERROR: not able to write a status file: ', statusfilename
print 'ERROR: ',x
raise
line='START: '+ time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
statusfile.flush()
os.chdir(workdir)
# -- WARNING: get the input files including the python modules BEFORE sys.path.insert()
# -- SINCE PYTHON 2.6 THERE WAS A SUBTLE CHANGE OF SEMANTICS IN THIS AREA
for f in input_sandbox:
getPackedInputSandbox(f)
# -- END OF MOVED CODE BLOCK
import sys
sys.path.insert(0, ###GANGADIR###)
sys.path.insert(0,os.path.join(os.getcwd(),PYTHON_DIR))
try:
import subprocess
except ImportError,x:
sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
import subprocess
try:
import tarfile
except ImportError,x:
sys.path.insert(0,###TARFILE_PYTHONPATH###)
import tarfile
for key,value in environment.iteritems():
os.environ[key] = value
outfile=file('stdout','w')
errorfile=file('stderr','w')
sys.stdout=file('./__syslog__','w')
sys.stderr=sys.stdout
###MONITORING_SERVICE###
monitor = createMonitoringObject()
monitor.start()
import subprocess
import time #datetime #disabled for python2.2 compatiblity
try:
child = subprocess.Popen(appscriptpath, shell=False, stdout=outfile, stderr=errorfile)
except OSError,x:
file('tt','w').close()
print >> statusfile, 'EXITCODE: %d'%-9999
print >> statusfile, 'FAILED: %s'%time.strftime('%a %b %d %H:%M:%S %Y') #datetime.datetime.utcnow().strftime('%a %b %d %H:%M:%S %Y')
print >> statusfile, 'PROBLEM STARTING THE APPLICATION SCRIPT: %s %s'%(appscriptpath,str(x))
statusfile.close()
sys.exit()
print >> statusfile, 'PID: %d'%child.pid
statusfile.flush()
result = -1
try:
while 1:
result = child.poll()
if result is not None:
break
outfile.flush()
errorfile.flush()
monitor.progress()
time.sleep(0.3)
finally:
monitor.progress()
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
monitor.stop(result)
outfile.flush()
errorfile.flush()
createOutputSandbox(outputpatterns,None,sharedoutputpath)
def printError(message):
errorfile.write(message + os.linesep)
errorfile.flush()
def printInfo(message):
outfile.write(message + os.linesep)
outfile.flush()
###OUTPUTUPLOADSPOSTPROCESSING###
outfile.close()
errorfile.close()
###OUTPUTSANDBOXPOSTPROCESSING###
line="EXITCODE: " + repr(result) + os.linesep
line+='STOP: '+time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
sys.exit()
"""
import inspect
script = script.replace('###INLINEMODULES###',inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###',getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__', getConfig('Output')['PostProcessLocationsFileName']], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###',getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###APPLICATION_NAME###',repr(job.application._name))
script = script.replace('###INPUT_SANDBOX###',repr(subjob_input_sandbox+master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###',repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###',repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###',repr(outputpatterns))
script = script.replace('###JOBID###',jobidRepr)
script = script.replace('###ENVIRONMENT###',repr(environment))
script = script.replace('###WORKDIR###',repr(workdir))
script = script.replace('###INPUT_DIR###',repr(job.getStringInputDir()))
script = script.replace('###MONITORING_SERVICE###',job.getMonitoringService().getWrapperScriptConstructorText())
self.workdir = workdir
script = script.replace('###GANGADIR###',repr(getConfig('System')['GANGA_PYTHONPATH']))
import Ganga.PACKAGE
script = script.replace('###SUBPROCESS_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('subprocess','syspath',force=True)))
script = script.replace('###TARFILE_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('tarfile','syspath',force=True)))
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__',script),executable=1)