def preparejob( self, jobconfig, master_input_sandbox ):
"""Prepare the script to create the job on the remote host"""
from Ganga.Utility import tempfile
workdir = tempfile.mkdtemp()
job = self.getJobObject()
script = """#!/usr/bin/env python
#-----------------------------------------------------
# This job wrapper script is automatically created by
# GANGA Remote backend handler.
#
# It controls:
# 1. unpack input sandbox
# 2. create the new job
# 3. submit it
#-----------------------------------------------------
import os,os.path,shutil,tempfile
import sys,popen2,time,traceback
import tarfile
############################################################################################
###INLINEMODULES###
############################################################################################
j = Job()
output_sandbox = ###OUTPUTSANDBOX###
input_sandbox = ###INPUTSANDBOX###
appexec = ###APPLICATIONEXEC###
appargs = ###APPLICATIONARGS###
back_end = ###BACKEND###
ganga_dir = ###GANGADIR###
code = ###CODE###
environment = ###ENVIRONMENT###
user_env = ###USERENV###
if user_env != None:
for env_var in user_env:
environment[env_var] = user_env[env_var]
j.outputsandbox = output_sandbox
j.backend = back_end
# Unpack the input sandboxes
shutil.move(os.path.expanduser(ganga_dir + "/__subjob_input_sbx__" + code), j.inputdir+"/__subjob_input_sbx__")
shutil.move(os.path.expanduser(ganga_dir + "/__master_input_sbx__" + code), j.inputdir+"/__master_input_sbx__")
# Add the files in the sandbox to the job
inputsbx = []
fullsbxlist = []
try:
tar = tarfile.open(j.inputdir+"/__master_input_sbx__")
filelist = tar.getnames()
print filelist
for f in filelist:
fullsbxlist.append( f )
inputsbx.append( j.inputdir + "/" + f )
except:
print "Unable to open master input sandbox"
try:
tar = tarfile.open(j.inputdir+"/__subjob_input_sbx__")
filelist = tar.getnames()
for f in filelist:
fullsbxlist.append( f )
inputsbx.append( j.inputdir + "/" + f )
except:
print "Unable to open subjob input sandbox"
# sort out the path of the exe
if appexec in fullsbxlist:
j.application = Executable ( exe = File(os.path.join(j.inputdir, appexec)), args = appargs, env = environment )
print "Script found: %s" % appexec
else:
j.application = Executable ( exe = appexec, args = appargs, env = environment )
j.inputsandbox = inputsbx
getPackedInputSandbox(j.inputdir+"/__subjob_input_sbx__", j.inputdir + "/.")
getPackedInputSandbox(j.inputdir+"/__master_input_sbx__", j.inputdir + "/.")
# submit the job
j.submit()
# Start pickle token
print "***_START_PICKLE_***"
# pickle the job
import pickle
print j.outputdir
print pickle.dumps(j._impl)
# print a finished token
print "***_END_PICKLE_***"
print "***_FINISHED_***"
"""
import inspect
import Ganga.Core.Sandbox as Sandbox
script = script.replace('###ENVIRONMENT###', repr(jobconfig.env) )
script = script.replace('###USERENV###', repr(self.environment) )
script = script.replace('###INLINEMODULES###', inspect.getsource(Sandbox.WNSandbox))
script = script.replace('###OUTPUTSANDBOX###', repr(jobconfig.outputbox))
script = script.replace('###APPLICATIONEXEC###',repr(os.path.basename(jobconfig.getExeString())))
script = script.replace('###APPLICATIONARGS###',repr(jobconfig.getArgStrings()))
# get a string describing the required backend
import StringIO
be_out = StringIO.StringIO()
job.backend.remote_backend.printTree( be_out, "copyable" )
be_str = be_out.getvalue()
script = script.replace('###BACKEND###', be_str)
script = script.replace('###GANGADIR###', repr(self.ganga_dir))
script = script.replace('###CODE###', repr(self._code))
sandbox_list = jobconfig.getSandboxFiles()
str_list = "[ "
for fname in sandbox_list:
str_list += "j.inputdir + '/' + " + repr(os.path.basename( fname.name ))
str_list += ", "
str_list += "j.inputdir + '/__master_input_sbx__' ]"
script = script.replace('###INPUTSANDBOX###', str_list)
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__.py',script),executable=0)
def preparejob( self, jobconfig, master_input_sandbox ):
"""Method for preparing job script"""
job = self.getJobObject()
inbox = job.createPackedInputSandbox( jobconfig.getSandboxFiles() )
inbox.extend( master_input_sandbox )
inpDir = job.getInputWorkspace().getPath()
outDir = job.getOutputWorkspace().getPath()
workdir = tempfile.mkdtemp()
self.workdir = workdir
exeString = repr( jobconfig.getExeString() )
argList = jobconfig.getArgStrings()
argString = " ".join( map( lambda x : "' \\'%s\\' '" % x, argList ) )
commandList = [
"#!/usr/bin/env python",
"# Interactive job wrapper created by Ganga",
"# %s" % ( time.strftime( "%c" ) ),
"",
inspect.getsource( Sandbox.WNSandbox ),
"import os",
"import sys",
"import time",
"",
"sys.path.insert( 0, '%s' )" % \
getConfig( "System" )[ "GANGA_PYTHONPATH" ],
"",
"statfileName = os.path.join( '%s', '__jobstatus__' )" % outDir,
"try:",
" statfile = open( statfileName, 'w' )",
"except IOError, x:",
" print 'ERROR: Unable to write status file: %s' % statfileName",
" print 'ERROR: ',x",
" raise",
"",
"idfileName = os.path.join( '%s', '__id__' )" % outDir,
"try:",
" idfile = open( idfileName, 'w' )",
"except IOError, x:",
" print 'ERROR: Unable to write id file: %s' % idfileName",
" print 'ERROR: ',x",
" raise",
"idfile.close()",
"",
"timeString = time.strftime"\
+ "( '%a %d %b %H:%M:%S %Y', time.gmtime( time.time() ) )",
"statfile.write( 'START: ' + timeString + os.linesep )",
"",
"os.chdir( '%s' )" % workdir,
"for inFile in %s:" % inbox,
" getPackedInputSandbox( inFile )",
"",
"for key, value in %s.iteritems():" % jobconfig.env,
" os.environ[ key ] = value",
"",
"pyCommandList = [",
" 'import os',",
" 'idfileName = \"%s\"' % idfileName,",
" 'idfile = open( idfileName, \"a\" )',",
" 'idfile.write( \"PID: \" + str( os.getppid() ) )',",
" 'idfile.flush()',",
" 'idfile.close()' ]",
"pyCommandString = ';'.join( pyCommandList )",
"",
"commandList = [",
" 'python -c \\\'%s\\\'' % pyCommandString,",
" 'exec ' " + exeString + " " + argString + "]",
"commandString = ';'.join( commandList )",
"",
"result = os.system( '%s' % commandString )",
"",
"createOutputSandbox( %s, None, '%s' )" % \
( jobconfig.outputbox, outDir ),
"",
"statfile.write( 'EXITCODE: ' + str( result >> 8 ) + os.linesep )",
"timeString = time.strftime"\
+ "( '%a %d %b %H:%M:%S %Y', time.gmtime( time.time() ) )",
"statfile.write( 'STOP: ' + timeString + os.linesep )",
"statfile.flush()",
"statfile.close()" ]
commandString = "\n".join( commandList )
return job.getInputWorkspace().writefile\
( FileBuffer( "__jobscript__", commandString), executable = 1 )
def preparejob(self,jobconfig,master_input_sandbox):
job = self.getJobObject()
#print str(job.backend_output_postprocess)
mon = job.getMonitoringService()
import Ganga.Core.Sandbox as Sandbox
subjob_input_sandbox = job.createPackedInputSandbox(jobconfig.getSandboxFiles()
+ Sandbox.getGangaModulesAsSandboxFiles(Sandbox.getDefaultModules())
+ Sandbox.getGangaModulesAsSandboxFiles(mon.getSandboxModules()))
appscriptpath = [jobconfig.getExeString()]+jobconfig.getArgStrings()
if self.nice:
appscriptpath = ['nice','-n %d'%self.nice] + appscriptpath
if self.nice < 0:
logger.warning('increasing process priority is often not allowed, your job may fail due to this')
sharedoutputpath=job.getOutputWorkspace().getPath()
outputpatterns=jobconfig.outputbox
environment=jobconfig.env
from Ganga.Utility import tempfile
workdir = tempfile.mkdtemp()
script= """#!/usr/bin/env python
import os,os.path,shutil,tempfile
import sys,time
import glob
import sys
# FIXME: print as DEBUG: to __syslog__ file
#print sys.path
#print os.environ['PATH']
#print sys.version
# bugfix #13314 : make sure that the wrapper (spawned process) is detached from Ganga session
# the process will not receive Control-C signals
# using fork and doing setsid() before exec would probably be a bit
# better (to avoid slim chance that the signal is propagated to this
# process before setsid is reached)
# this is only enabled if the first argument is 'subprocess' in order to enable
# running this script by hand from outside ganga (which is sometimes useful)
if len(sys.argv)>1 and sys.argv[1] == 'subprocess':
os.setsid()
############################################################################################
###INLINEMODULES###
############################################################################################
input_sandbox = ###INPUT_SANDBOX###
sharedoutputpath= ###SHAREDOUTPUTPATH###
outputpatterns = ###OUTPUTPATTERNS###
appscriptpath = ###APPSCRIPTPATH###
environment = ###ENVIRONMENT###
workdir = ###WORKDIR###
statusfilename = os.path.join(sharedoutputpath,'__jobstatus__')
try:
statusfile=file(statusfilename,'w')
except IOError,x:
print 'ERROR: not able to write a status file: ', statusfilename
print 'ERROR: ',x
raise
line='START: '+ time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
statusfile.flush()
os.chdir(workdir)
# -- WARNING: get the input files including the python modules BEFORE sys.path.insert()
# -- SINCE PYTHON 2.6 THERE WAS A SUBTLE CHANGE OF SEMANTICS IN THIS AREA
for f in input_sandbox:
getPackedInputSandbox(f)
# -- END OF MOVED CODE BLOCK
import sys
sys.path.insert(0, ###GANGADIR###)
sys.path.insert(0,os.path.join(os.getcwd(),PYTHON_DIR))
try:
import subprocess
except ImportError,x:
sys.path.insert(0,###SUBPROCESS_PYTHONPATH###)
import subprocess
try:
import tarfile
except ImportError,x:
sys.path.insert(0,###TARFILE_PYTHONPATH###)
import tarfile
for key,value in environment.iteritems():
os.environ[key] = value
outfile=file('stdout','w')
errorfile=file('stderr','w')
sys.stdout=file('./__syslog__','w')
sys.stderr=sys.stdout
###MONITORING_SERVICE###
monitor = createMonitoringObject()
monitor.start()
import subprocess
import time #datetime #disabled for python2.2 compatiblity
try:
child = subprocess.Popen(appscriptpath, shell=False, stdout=outfile, stderr=errorfile)
except OSError,x:
file('tt','w').close()
print >> statusfile, 'EXITCODE: %d'%-9999
print >> statusfile, 'FAILED: %s'%time.strftime('%a %b %d %H:%M:%S %Y') #datetime.datetime.utcnow().strftime('%a %b %d %H:%M:%S %Y')
print >> statusfile, 'PROBLEM STARTING THE APPLICATION SCRIPT: %s %s'%(appscriptpath,str(x))
statusfile.close()
sys.exit()
print >> statusfile, 'PID: %d'%child.pid
statusfile.flush()
result = -1
try:
while 1:
result = child.poll()
if result is not None:
break
outfile.flush()
errorfile.flush()
monitor.progress()
time.sleep(0.3)
finally:
monitor.progress()
sys.stdout=sys.__stdout__
sys.stderr=sys.__stderr__
monitor.stop(result)
outfile.flush()
errorfile.flush()
createOutputSandbox(outputpatterns,None,sharedoutputpath)
def printError(message):
errorfile.write(message + os.linesep)
errorfile.flush()
def printInfo(message):
outfile.write(message + os.linesep)
outfile.flush()
###OUTPUTUPLOADSPOSTPROCESSING###
outfile.close()
errorfile.close()
###OUTPUTSANDBOXPOSTPROCESSING###
line="EXITCODE: " + repr(result) + os.linesep
line+='STOP: '+time.strftime('%a %b %d %H:%M:%S %Y',time.gmtime(time.time())) + os.linesep
statusfile.writelines(line)
sys.exit()
"""
import inspect
script = script.replace('###INLINEMODULES###',inspect.getsource(Sandbox.WNSandbox))
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForOutputSandbox, getWNCodeForOutputPostprocessing
from Ganga.Utility.Config import getConfig
jobidRepr = repr(job.getFQID('.'))
script = script.replace('###OUTPUTSANDBOXPOSTPROCESSING###',getWNCodeForOutputSandbox(job, ['stdout', 'stderr', '__syslog__', getConfig('Output')['PostProcessLocationsFileName']], jobidRepr))
script = script.replace('###OUTPUTUPLOADSPOSTPROCESSING###',getWNCodeForOutputPostprocessing(job, ''))
script = script.replace('###APPLICATION_NAME###',repr(job.application._name))
script = script.replace('###INPUT_SANDBOX###',repr(subjob_input_sandbox+master_input_sandbox))
script = script.replace('###SHAREDOUTPUTPATH###',repr(sharedoutputpath))
script = script.replace('###APPSCRIPTPATH###',repr(appscriptpath))
script = script.replace('###OUTPUTPATTERNS###',repr(outputpatterns))
script = script.replace('###JOBID###',jobidRepr)
script = script.replace('###ENVIRONMENT###',repr(environment))
script = script.replace('###WORKDIR###',repr(workdir))
script = script.replace('###INPUT_DIR###',repr(job.getStringInputDir()))
script = script.replace('###MONITORING_SERVICE###',job.getMonitoringService().getWrapperScriptConstructorText())
self.workdir = workdir
script = script.replace('###GANGADIR###',repr(getConfig('System')['GANGA_PYTHONPATH']))
import Ganga.PACKAGE
script = script.replace('###SUBPROCESS_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('subprocess','syspath',force=True)))
script = script.replace('###TARFILE_PYTHONPATH###',repr(Ganga.PACKAGE.setup.getPackagePath2('tarfile','syspath',force=True)))
return job.getInputWorkspace().writefile(FileBuffer('__jobscript__',script),executable=1)