def getWNCodeForOutputPostprocessing(job, indent):
# dict containing the list of outputfiles that need to be processed on the
# WN for every file type
outputFilesProcessedOnWN = {}
patternsToZip = []
if len(job.outputfiles) == 0:
return ""
else:
for outputFile in job.outputfiles:
outputfileClassName = getName(outputFile)
backendClassName = getName(job.backend)
if outputFile.compressed:
patternsToZip.append(outputFile.namePattern)
if outputfileClassName not in outputFilesProcessedOnWN.keys():
outputFilesProcessedOnWN[outputfileClassName] = []
if outputFilePostProcessingOnWN(job, outputfileClassName):
outputFilesProcessedOnWN[outputfileClassName].append(outputFile)
if not patternsToZip:
if not any(outputFilesProcessedOnWN.values()):
return ""
logger.debug("Process: '%s' on WN" % str(outputFilePostProcessingOnWN))
shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
if os.path.isfile(currentFile):
os.system("gzip %s" % currentFile)
postprocesslocations = open(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'a+')
"""
from Ganga.GPIDev.Lib.File import FileUtils
shortScript = FileUtils.indentScript(shortScript, '###INDENT###')
insertScript = shortScript
insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])
for outputFileName in outputFilesProcessedOnWN.keys():
if len(outputFilesProcessedOnWN[outputFileName]) > 0:
insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')
insertScript += """\n
###INDENT###postprocesslocations.close()
"""
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def getWNCodeForDownloadingInputFiles(job, indent):
"""
Generate the code to be run on the WN to download input files
"""
from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset
if (
job.inputfiles is None
or len(job.inputfiles) == 0
and (not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or not job.inputdata.treat_as_inputfiles))
):
return ""
insertScript = """\n
"""
# first, go over any LocalFiles in GangaDatasets to be transferred
# The LocalFiles in inputfiles have already been dealt with
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
for inputFile in job.inputdata.files:
inputfileClassName = getName(inputFile)
if inputfileClassName == "LocalFile":
# special case for LocalFile
if getName(job.backend) in ["Localhost", "Batch", "LSF", "Condor", "PBS"]:
# create symlink
shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
os.symlink(f, os.path.basename(f))
"""
shortScript = FileUtils.indentScript(shortScript, "###INDENT####")
insertScript += shortScript
insertScript = insertScript.replace("###FILELIST###", "%s" % inputFile.getFilenameList())
# if GangaDataset is used, check if they want the inputfiles transferred
inputfiles_list = job.inputfiles
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
inputfiles_list += job.inputdata.files
for inputFile in inputfiles_list:
inputfileClassName = getName(inputFile)
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
for subfile in inputFile.subfiles:
insertScript += subfile.getWNScriptDownloadCommand(indent)
else:
insertScript += inputFile.getWNScriptDownloadCommand(indent)
insertScript = insertScript.replace("###INDENT###", indent)
return insertScript
def getWNCodeForOutputPostprocessing(job, indent):
# dict containing the list of outputfiles that need to be processed on the
# WN for every file type
outputFilesProcessedOnWN = {}
patternsToZip = []
if len(job.outputfiles) == 0:
return ""
else:
for outputFile in job.outputfiles:
outputfileClassName = getName(outputFile)
backendClassName = getName(job.backend)
if outputFile.compressed:
if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
patternsToZip.append(outputFile.namePattern)
elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName):
patternsToZip.append(outputFile.namePattern)
elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
patternsToZip.append(outputFile.namePattern)
if outputfileClassName not in outputFilesProcessedOnWN.keys():
outputFilesProcessedOnWN[outputfileClassName] = []
if outputFilePostProcessingOnWN(job, outputfileClassName):
outputFilesProcessedOnWN[
outputfileClassName].append(outputFile)
shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
os.system("gzip %s" % currentFile)
postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w')
"""
shortScript = FileUtils.indentScript(shortScript, '###INDENT###')
insertScript = shortScript
insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])
for outputFileName in outputFilesProcessedOnWN.keys():
if len(outputFilesProcessedOnWN[outputFileName]) > 0:
insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')
insertScript += """\n
###INDENT###postprocesslocations.close()
"""
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def getWNCodeForOutputPostprocessing(job, indent):
# dict containing the list of outputfiles that need to be processed on the
# WN for every file type
outputFilesProcessedOnWN = {}
patternsToZip = []
if len(job.outputfiles) == 0:
return ""
else:
for outputFile in job.outputfiles:
outputfileClassName = getName(outputFile)
backendClassName = getName(job.backend)
if outputFile.compressed:
if outputfileClassName == 'LocalFile' and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
patternsToZip.append(outputFile.namePattern)
elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnWN(job, outputfileClassName):
patternsToZip.append(outputFile.namePattern)
elif outputfileClassName != 'LocalFile' and outputFilePostProcessingOnClient(job, outputfileClassName) and backendClassName not in ['Localhost', 'LSF', 'Interactive']:
patternsToZip.append(outputFile.namePattern)
if outputfileClassName not in outputFilesProcessedOnWN.keys():
outputFilesProcessedOnWN[outputfileClassName] = []
if outputFilePostProcessingOnWN(job, outputfileClassName):
outputFilesProcessedOnWN[
outputfileClassName].append(outputFile)
shortScript = """\n
import os, glob
for patternToZip in ###PATTERNSTOZIP###:
for currentFile in glob.glob(os.path.join(os.getcwd(),patternToZip)):
os.system("gzip %s" % currentFile)
postprocesslocations = file(os.path.join(os.getcwd(), '###POSTPROCESSLOCATIONSFILENAME###'), 'w')
"""
shortScript = FileUtils.indentScript(shortScript, '###INDENT###')
insertScript = shortScript
insertScript = insertScript.replace('###PATTERNSTOZIP###', str(patternsToZip))
insertScript = insertScript.replace('###POSTPROCESSLOCATIONSFILENAME###', getConfig('Output')['PostProcessLocationsFileName'])
for outputFileName in outputFilesProcessedOnWN.keys():
if len(outputFilesProcessedOnWN[outputFileName]) > 0:
insertScript += outputFilesProcessedOnWN[outputFileName][0].getWNInjectedScript(outputFilesProcessedOnWN[outputFileName], indent, patternsToZip, 'postprocesslocations')
insertScript += """\n
###INDENT###postprocesslocations.close()
"""
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def getWNCodeForDownloadingInputFiles(job, indent):
"""
Generate the code to be run on the WN to download input files
"""
from Ganga.GPIDev.Lib.Dataset.GangaDataset import GangaDataset
if job.inputfiles is None or len(job.inputfiles) == 0 and\
(not job.inputdata or ((not isType(job.inputdata, GangaDataset)) or\
not job.inputdata.treat_as_inputfiles )):
return ""
insertScript = """\n
"""
# first, go over any LocalFiles in GangaDatasets to be transferred
# The LocalFiles in inputfiles have already been dealt with
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
for inputFile in job.inputdata.files:
inputfileClassName = getName(inputFile)
if inputfileClassName == "LocalFile":
# special case for LocalFile
if getName(job.backend) in ['Localhost', 'Batch', 'LSF', 'Condor', 'PBS']:
# create symlink
shortScript = """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
os.symlink(f, os.path.basename(f))
"""
shortScript = FileUtils.indentScript(shortScript, '###INDENT####')
insertScript += shortScript
insertScript = insertScript.replace('###FILELIST###', "%s" % inputFile.getFilenameList())
# if GangaDataset is used, check if they want the inputfiles transferred
inputfiles_list = job.inputfiles
if job.inputdata and isType(job.inputdata, GangaDataset) and job.inputdata.treat_as_inputfiles:
inputfiles_list += job.inputdata.files
for inputFile in inputfiles_list:
inputfileClassName = getName(inputFile)
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
for subfile in inputFile.subfiles:
insertScript += subfile.getWNScriptDownloadCommand(indent)
else:
insertScript += inputFile.getWNScriptDownloadCommand(indent)
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def getWNCodeForDownloadingInputFiles(job, indent):
"""
Generate the code to be run on the WN to download input files
"""
if len(job.inputfiles) == 0 and (not job.inputdata or job.inputdata._name != "GangaDataset" or not job.inputdata.treat_as_inputfiles):
return ""
insertScript = """\n
"""
# first, go over any LocalFiles in GangaDatasets to be transferred
# The LocalFiles in inputfiles have already been dealt with
if job.inputdata and job.inputdata._name == "GangaDataset" and job.inputdata.treat_as_inputfiles:
for inputFile in job.inputdata.files:
inputfileClassName = stripProxy(inputFile).__class__.__name__
if inputfileClassName == "LocalFile":
# special case for LocalFile
if stripProxy(job.backend).__class__.__name__ in ['Localhost', 'Batch', 'LSF', 'Condor', 'PBS']:
# create symlink
shortScript += """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
os.symlink(f, os.path.basename(f))
"""
shortScript = FileUtils.indentScript(shortScript, '###INDENT####')
insertScript += shortScript
insertScript = insertScript.replace('###FILELIST###', "%s" % inputFile.getFilenameList())
# if GangaDataset is used, check if they want the inputfiles transferred
inputfiles_list = job.inputfiles
if job.inputdata and job.inputdata._name == "GangaDataset" and job.inputdata.treat_as_inputfiles:
inputfiles_list += job.inputdata.files
for inputFile in inputfiles_list:
inputfileClassName = stripProxy(inputFile).__class__.__name__
if outputFilePostProcessingOnWN(job, inputfileClassName):
inputFile.processWildcardMatches()
if inputFile.subfiles:
for subfile in inputFile.subfiles:
insertScript += subfile.getWNScriptDownloadCommand(indent)
else:
insertScript += inputFile.getWNScriptDownloadCommand(indent)
insertScript = insertScript.replace('###INDENT###', indent)
return insertScript
def _WN_wildcard_script(self, namePattern, lfnBase, compressed):
wildcard_str = """
for f in glob.glob('###NAME_PATTERN###'):
processes.append(uploadFile(os.path.basename(f), '###LFN_BASE###', ###COMPRESSED###, '###NAME_PATTERN###'))
"""
wildcard_str = FileUtils.indentScript(wildcard_str, '###INDENT###')
replace_dict = { '###NAME_PATTERN###' : namePattern,
'###LFN_BASE###' : lfnBase,
'###COMPRESSED###' : compressed }
for k, v in replace_dict.iteritems():
wildcard_str = wildcard_str.replace(str(k), str(v))
return wildcard_str
def _WN_wildcard_script(self, namePattern, lfnBase, compressed):
wildcard_str = """
for f in glob.glob('###NAME_PATTERN###'):
processes.append(uploadFile(os.path.basename(f), '###LFN_BASE###', ###COMPRESSED###, '###NAME_PATTERN###'))
"""
wildcard_str = FileUtils.indentScript(wildcard_str, '###INDENT###')
replace_dict = { '###NAME_PATTERN###' : namePattern,
'###LFN_BASE###' : lfnBase,
'###COMPRESSED###' : compressed }
for k, v in replace_dict.iteritems():
wildcard_str = wildcard_str.replace(str(k), str(v))
return wildcard_str
def getWNScriptDownloadCommand(self, indent):
# create symlink
shortScript = """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
if not os.path.exists(os.path.basename(f)):
os.symlink(f, os.path.basename(f))
"""
from Ganga.GPIDev.Lib.File import FileUtils
shortScript = FileUtils.indentScript(shortScript, '###INDENT###')
shortScript = shortScript.replace('###FILELIST###', "%s" % self.getFilenameList())
return shortScript
def getWNScriptDownloadCommand(self, indent):
# create symlink
shortScript = """
# create symbolic links for LocalFiles
for f in ###FILELIST###:
if not os.path.exists(os.path.basename(f)):
os.symlink(f, os.path.basename(f))
"""
from Ganga.GPIDev.Lib.File import FileUtils
shortScript = FileUtils.indentScript(shortScript, '###INDENT###')
shortScript = shortScript.replace('###FILELIST###',
"%s" % self.getFilenameList())
return shortScript
def _WN_wildcard_script(self, namePattern, lfnBase, compressed):
wildcard_str = """
import glob, hashlib
for f in glob.glob('###NAME_PATTERN###'):
processes.append(uploadFile(os.path.basename(f), '###LFN_BASE###', ###COMPRESSED###, '###NAME_PATTERN###'))
"""
from Ganga.GPIDev.Lib.File import FileUtils
wildcard_str = FileUtils.indentScript(wildcard_str, "###INDENT###")
replace_dict = {"###NAME_PATTERN###": namePattern, "###LFN_BASE###": lfnBase, "###COMPRESSED###": compressed}
for k, v in replace_dict.iteritems():
wildcard_str = wildcard_str.replace(str(k), str(v))
return wildcard_str
