def get_output(self, job):
'''Builds lists of output files and output data.'''
outputdata = [
f.namePattern for f in job.outputfiles
if outputFilePostProcessingOnWN(job, getName(f))
]
outsandbox = [
f.namePattern for f in job.outputfiles
if not outputFilePostProcessingOnWN(job, getName(f))
]
# if user put any files in both, remove them from the sandbox
for f in outsandbox:
if outputdata.count(f) != 0:
outsandbox.remove(f)
msg = 'User placed the file %s in both the outputsandbox and '
msg += 'outputdata. It will be removed from the sandbox.'
logger.warning(msg, f)
gaudi_outsandbox, gaudi_outputdata = self.get_output_files()
# handle (as best we can) any user supplied wildcards
datalist = [] # files in sandbox that match pattern in data
for f in outputdata:
datalist += fnmatch.filter(gaudi_outsandbox, f)
sandlist = [] # files in data that match sandbox pattern
for f in outsandbox:
sandlist += fnmatch.filter(gaudi_outputdata, f)
datadatalist = [] # files in data that match patterns in data
for f in outputdata:
datadatalist += fnmatch.filter(gaudi_outputdata, f)
# files in sandbox which match patterns in data -> data
for f in datalist:
gaudi_outputdata.append(f)
if f in gaudi_outsandbox:
gaudi_outsandbox.remove(f)
# files in data which match patterns in sandbox but not data -> sandbox
for f in sandlist:
if datalist.count(f) == 0 and datadatalist.count(f) == 0:
gaudi_outsandbox.append(f)
if f in gaudi_outputdata:
gaudi_outputdata.remove(f)
outsandbox += gaudi_outsandbox
outputdata += gaudi_outputdata
return unique(outsandbox), unique(outputdata)
def get_output(self, job):
'''Builds lists of output files and output data.'''
outputdata = [f.namePattern for f in job.outputfiles if outputFilePostProcessingOnWN(job, getName(f)) ]
outsandbox = [f.namePattern for f in job.outputfiles if not outputFilePostProcessingOnWN(job, getName(f)) ]
# if user put any files in both, remove them from the sandbox
for f in outsandbox:
if outputdata.count(f) != 0:
outsandbox.remove(f)
msg = 'User placed the file %s in both the outputsandbox and '
msg += 'outputdata. It will be removed from the sandbox.'
logger.warning(msg, f)
gaudi_outsandbox, gaudi_outputdata = self.get_output_files()
# handle (as best we can) any user supplied wildcards
datalist = [] # files in sandbox that match pattern in data
for f in outputdata:
datalist += fnmatch.filter(gaudi_outsandbox, f)
sandlist = [] # files in data that match sandbox pattern
for f in outsandbox:
sandlist += fnmatch.filter(gaudi_outputdata, f)
datadatalist = [] # files in data that match patterns in data
for f in outputdata:
datadatalist += fnmatch.filter(gaudi_outputdata, f)
# files in sandbox which match patterns in data -> data
for f in datalist:
gaudi_outputdata.append(f)
if f in gaudi_outsandbox:
gaudi_outsandbox.remove(f)
# files in data which match patterns in sandbox but not data -> sandbox
for f in sandlist:
if datalist.count(f) == 0 and datadatalist.count(f) == 0:
gaudi_outsandbox.append(f)
if f in gaudi_outputdata:
gaudi_outputdata.remove(f)
outsandbox += gaudi_outsandbox
outputdata += gaudi_outputdata
return unique(outsandbox), unique(outputdata)
def preparejob(self, jobconfig, master_input_sandbox):
"""Method for preparing job script"""
job = self.getJobObject()
inputfiles = jobconfig.getSandboxFiles()
inbox = job.createPackedInputSandbox(inputfiles)
inbox.extend(master_input_sandbox)
inpDir = job.getInputWorkspace(create=True).getPath()
outDir = job.getOutputWorkspace(create=True).getPath()
workdir = tempfile.mkdtemp()
self.workdir = workdir
exeString = jobconfig.getExeString()
argList = jobconfig.getArgStrings()
argString = " ".join(map(lambda x: " %s " % x, argList))
outputSandboxPatterns = jobconfig.outputbox
patternsToZip = []
wnCodeForPostprocessing = ''
wnCodeToDownloadInputFiles = ''
if (len(job.outputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
(outputSandboxPatterns,
patternsToZip) = getOutputSandboxPatternsForInteractive(job)
wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
getWNCodeForOutputPostprocessing(job, '')
all_inputfiles = [this_file for this_file in job.inputfiles]
if job.master: all_inputfiles.extend([this_file for this_file in job.master.inputfiles])
wnCodeToDownloadInputFiles = ''
if(len(all_inputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN
for inputFile in all_inputfiles:
inputfileClassName = getName(inputFile)
logger.debug("name: %s" % inputfileClassName)
logger.debug("result: %s" % str(outputFilePostProcessingOnWN(job, inputfileClassName)))
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
getfromFile = False
for subfile in inputFile.subfiles:
wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand('')
else:
getfromFile = True
else:
getFromFile = True
if getFromFile:
wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand('')
wnCodeToDownloadInputData = ''
if job.inputdata and (len(job.inputdata) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles
wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(job, '')
import inspect
replace_dict = {
'###CONSTRUCT_TIME###' : (time.strftime("%c")),
'###WNSANDBOX_SOURCE###' : inspect.getsource(Sandbox.WNSandbox),
'###GANGA_PYTHONPATH###' : getConfig("System")["GANGA_PYTHONPATH"],
'###OUTPUTDIR###' : outDir,
'###WORKDIR###' : workdir,
'###IN_BOX###' : inbox,
'###WN_INPUTFILES###' : wnCodeToDownloadInputFiles,
'###WN_INPUTDATA###' : wnCodeToDownloadInputData,
'###JOBCONFIG_ENV###' : jobconfig.env if jobconfig.env is not None else dict(),
'###EXE_STRING###' : exeString,
'###ARG_STRING###' : argString,
'###WN_POSTPROCESSING###' : wnCodeForPostprocessing,
'###PATTERNS_TO_ZIP###' : patternsToZip,
'###OUTPUT_SANDBOX_PATTERNS###' : outputSandboxPatterns
}
script_location = os.path.join(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe()))),
'InteractiveScriptTemplate.py')
from Ganga.GPIDev.Lib.File import FileUtils
commandString = FileUtils.loadScript(script_location, '')
for k, v in replace_dict.iteritems():
commandString = commandString.replace(str(k), str(v))
return job.getInputWorkspace().writefile(FileBuffer("__jobscript__", commandString), executable=1)
def preparejob(self, jobconfig, master_input_sandbox):
"""Method for preparing job script"""
job = self.getJobObject()
from Ganga.GPIDev.Lib.File import File
from Ganga.Core.Sandbox.WNSandbox import PYTHON_DIR
import Ganga.Utility.files
import inspect
fileutils = File(inspect.getsourcefile(Ganga.Utility.files),
subdir=PYTHON_DIR)
inputfiles = jobconfig.getSandboxFiles() + [fileutils]
inbox = job.createPackedInputSandbox(inputfiles)
inbox.extend(master_input_sandbox)
inpDir = job.getInputWorkspace(create=True).getPath()
outDir = job.getOutputWorkspace(create=True).getPath()
workdir = tempfile.mkdtemp()
self.workdir = workdir
exeString = jobconfig.getExeString()
argList = jobconfig.getArgStrings()
argString = " ".join(map(lambda x: " %s " % x, argList))
outputSandboxPatterns = jobconfig.outputbox
patternsToZip = []
wnCodeForPostprocessing = ''
wnCodeToDownloadInputFiles = ''
if (len(job.outputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getOutputSandboxPatternsForInteractive, getWNCodeForOutputPostprocessing
(outputSandboxPatterns,
patternsToZip) = getOutputSandboxPatternsForInteractive(job)
wnCodeForPostprocessing = 'def printError(message):pass\ndef printInfo(message):pass' + \
getWNCodeForOutputPostprocessing(job, '')
all_inputfiles = [this_file for this_file in job.inputfiles]
if job.master is not None:
all_inputfiles.extend(
[this_file for this_file in job.master.inputfiles])
wnCodeToDownloadInputFiles = ''
if (len(all_inputfiles) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import outputFilePostProcessingOnWN
for inputFile in all_inputfiles:
inputfileClassName = getName(inputFile)
logger.debug("name: %s" % inputfileClassName)
logger.debug(
"result: %s" %
str(outputFilePostProcessingOnWN(job, inputfileClassName)))
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
getfromFile = False
for subfile in inputFile.subfiles:
wnCodeToDownloadInputFiles += subfile.getWNScriptDownloadCommand(
'')
else:
getfromFile = True
else:
getFromFile = True
if getFromFile:
wnCodeToDownloadInputFiles += inputFile.getWNScriptDownloadCommand(
'')
wnCodeToDownloadInputData = ''
if job.inputdata and (len(job.inputdata) > 0):
from Ganga.GPIDev.Lib.File.OutputFileManager import getWNCodeForDownloadingInputFiles
wnCodeToDownloadInputData = getWNCodeForDownloadingInputFiles(
job, '')
import inspect
replace_dict = {
'###CONSTRUCT_TIME###': (time.strftime("%c")),
'###WNSANDBOX_SOURCE###':
inspect.getsource(Sandbox.WNSandbox),
'###GANGA_PYTHONPATH###':
getConfig("System")["GANGA_PYTHONPATH"],
'###OUTPUTDIR###':
outDir,
'###WORKDIR###':
workdir,
'###IN_BOX###':
inbox,
'###WN_INPUTFILES###':
wnCodeToDownloadInputFiles,
'###WN_INPUTDATA###':
wnCodeToDownloadInputData,
'###JOBCONFIG_ENV###':
jobconfig.env if jobconfig.env is not None else dict(),
'###EXE_STRING###':
exeString,
'###ARG_STRING###':
argString,
'###WN_POSTPROCESSING###':
wnCodeForPostprocessing,
'###PATTERNS_TO_ZIP###':
patternsToZip,
'###OUTPUT_SANDBOX_PATTERNS###':
outputSandboxPatterns
}
script_location = os.path.join(
os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
'InteractiveScriptTemplate.py.template')
from Ganga.GPIDev.Lib.File import FileUtils
commandString = FileUtils.loadScript(script_location, '')
for k, v in replace_dict.iteritems():
commandString = commandString.replace(str(k), str(v))
return job.getInputWorkspace().writefile(FileBuffer(
"__jobscript__", commandString),
executable=1)