def updateRunCommandList(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO, hasInput, prodDBlockToken):
""" update the run command list if --directIn is no longer needed """
# the method is using the file state dictionary
# remove later
dumpFileStates(pworkdir, jobId, ftype="input")
# remove any instruction regarding tag file creation for event service jobs
_runCommandList = []
for cmd in runCommandList:
if "--createTAGFileForES" in cmd:
cmd = cmd.replace("--createTAGFileForES","")
_runCommandList.append(cmd)
runCommandList = _runCommandList
# no need to continue if no input files
if not hasInput:
return runCommandList
# are there only copy_to_scratch transfer modes in the file state dictionary?
# if so, remove any lingering --directIn instruction
only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
if only_copy_to_scratch or 'local' in prodDBlockToken:
# if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
_runCommandList = []
if only_copy_to_scratch:
tolog("There are only copy_to_scratch transfer modes in file state dictionary")
for cmd in runCommandList:
# remove the --directIn string if present
if "--directIn" in cmd:
tolog("(Removing --directIn instruction from run command since it is not needed)")
cmd = cmd.replace("--directIn", "")
# remove the --useFileStager string if present
if "--useFileStager" in cmd:
tolog("(Removing --useFileStager instruction from run command since it is not needed)")
cmd = cmd.replace("--useFileStager", "")
# remove additional run options if creation of TURL based PFC failed
if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
if "--usePFCTurl" in cmd:
tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
cmd = cmd.replace(" --usePFCTurl", "")
if not "--lfcHost" in cmd and analysisJob:
tolog("Adding lfcHost to run command")
cmd += ' --lfcHost %s' % (readpar('lfchost'))
tolog("Updated run command: %s" % (cmd))
_runCommandList.append(cmd)
else:
tolog("Nothing to update in run command list related to copy-to-scratch")
_runCommandList = runCommandList
# was FAX used as primary site mover in combination with direct I/O?
if usedFAXandDirectIO == True:
tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
_runCommandList2 = []
for cmd in _runCommandList:
# remove the --lfcHost
if "--lfcHost" in cmd:
_lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
cmd = cmd.replace(_lfcHost, '')
tolog("(Removed the LFC host:%s)" % (_lfcHost))
# remove the --oldPrefix
if "--oldPrefix" in cmd:
pattern = "(\-\-oldPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --oldPrefix pattern)")
# remove the --newPrefix
if "--newPrefix" in cmd:
pattern = "(\-\-newPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --newPrefix pattern)")
# add the --usePFCTurl if not there already
if not "--usePFCTurl" in cmd and analysisJob:
cmd += " --usePFCTurl"
tolog("(Added --usePFCTurl)")
tolog("Updated run command: %s" % (cmd))
_runCommandList2.append(cmd)
_runCommandList = _runCommandList2
### new movers quick integration: reuse usedFAXandDirectIO variable with special meaning
### to avoid any LFC and prefixes lookups in transformation scripts
### since new movers already form proper pfn values
### proper workflow is required: to be reimplemented later
if usedFAXandDirectIO == 'newmover' or usedFAXandDirectIO == 'newmover-directaccess':
tolog("updateRunCommandList(): use new movers logic")
tolog("updateRunCommandList(): remove to be deprecated options (--lfcHost, --oldPrefix, --newPrefix) from command list")
tolog("updateRunCommandList(): force to set --usePFCTurl")
tolog("updateRunCommandList(): check directaccess mode if need (--directIn)")
tolog("current runCommandList=%s" % _runCommandList)
_runCommandList2 = []
for cmd in _runCommandList:
# remove the --lfcHost, --oldPrefix, --newPrefix
# add --usePFCTurl
if "--lfcHost" in cmd:
cmd = removePattern(cmd, "(\-\-lfcHost\ \S+)")
tolog("(Removed the --lfcHost)")
if "--oldPrefix" in cmd:
pattern = "(\-\-oldPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --oldPrefix pattern)")
if "--newPrefix" in cmd:
pattern = "(\-\-newPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --newPrefix pattern)")
if "--usePFCTurl" not in cmd and analysisJob:
cmd += " --usePFCTurl"
tolog("(Added --usePFCTurl)")
# add --directIn if need
if usedFAXandDirectIO == 'newmover-directaccess':
if "--directIn" not in cmd and analysisJob:
cmd += " --directIn"
tolog("(Added --directIn)")
tolog("Updated run command: %s" % cmd)
_runCommandList2.append(cmd)
_runCommandList = _runCommandList2
tolog("Dumping final input file states")
dumpFileStates(pworkdir, jobId, ftype="input")
return _runCommandList
def updateRunCommandList(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO, hasInput, prodDBlockToken):
""" update the run command list if --directIn is no longer needed """
# the method is using the file state dictionary
# remove later
dumpFileStates(pworkdir, jobId, ftype="input")
# remove any instruction regarding tag file creation for event service jobs
_runCommandList = []
for cmd in runCommandList:
if "--createTAGFileForES" in cmd:
cmd = cmd.replace("--createTAGFileForES","")
_runCommandList.append(cmd)
runCommandList = _runCommandList
# no need to continue if no input files
if not hasInput:
return runCommandList
# are there only copy_to_scratch transfer modes in the file state dictionary?
# if so, remove any lingering --directIn instruction
only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
if only_copy_to_scratch or 'local' in prodDBlockToken:
# if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
_runCommandList = []
if only_copy_to_scratch:
tolog("There are only copy_to_scratch transfer modes in file state dictionary")
for cmd in runCommandList:
# remove the --directIn string if present
if "--directIn" in cmd:
tolog("(Removing --directIn instruction from run command since it is not needed)")
cmd = cmd.replace("--directIn", "")
# remove the --useFileStager string if present
if "--useFileStager" in cmd:
tolog("(Removing --useFileStager instruction from run command since it is not needed)")
cmd = cmd.replace("--useFileStager", "")
# remove additional run options if creation of TURL based PFC failed
if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
if "--usePFCTurl" in cmd:
tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
cmd = cmd.replace(" --usePFCTurl", "")
if not "--lfcHost" in cmd and analysisJob:
tolog("Adding lfcHost to run command")
cmd += ' --lfcHost %s' % (readpar('lfchost'))
tolog("Updated run command: %s" % (cmd))
_runCommandList.append(cmd)
else:
tolog("Nothing to update in run command list related to copy-to-scratch")
_runCommandList = runCommandList
# was FAX used as primary site mover in combination with direct I/O?
if usedFAXandDirectIO == True:
tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
_runCommandList2 = []
for cmd in _runCommandList:
# remove the --lfcHost
if "--lfcHost" in cmd:
_lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
cmd = cmd.replace(_lfcHost, '')
tolog("(Removed the LFC host:%s)" % (_lfcHost))
# remove the --oldPrefix
if "--oldPrefix" in cmd:
pattern = "(\-\-oldPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --oldPrefix pattern)")
# remove the --newPrefix
if "--newPrefix" in cmd:
pattern = "(\-\-newPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --newPrefix pattern)")
# add the --usePFCTurl if not there already
if not "--usePFCTurl" in cmd and analysisJob:
cmd += " --usePFCTurl"
tolog("(Added --usePFCTurl)")
tolog("Updated run command: %s" % (cmd))
_runCommandList2.append(cmd)
_runCommandList = _runCommandList2
### new movers quick integration: reuse usedFAXandDirectIO variable with special meaning
### to avoid any LFC and prefixes lookups in transformation scripts
### since new movers already form proper pfn values
### proper workflow is required: to be reimplemented later
if usedFAXandDirectIO == 'newmover' or usedFAXandDirectIO == 'newmover-directaccess':
_runCommandList2 = []
for cmd in _runCommandList:
# remove the --lfcHost, --oldPrefix, --newPrefix
# add --usePFCTurl
if "--lfcHost" in cmd:
cmd = removePattern(cmd, "(\-\-lfcHost\ \S+)")
tolog("(Removed the --lfcHost)")
if "--oldPrefix" in cmd:
pattern = "(\-\-oldPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --oldPrefix pattern)")
if "--newPrefix" in cmd:
pattern = "(\-\-newPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --newPrefix pattern)")
if "--usePFCTurl" not in cmd and analysisJob:
cmd += " --usePFCTurl"
tolog("(Added --usePFCTurl)")
# add --directIn if need
if usedFAXandDirectIO == 'newmover-directaccess':
if "--directIn" not in cmd and analysisJob:
cmd += " --directIn"
tolog("(Added --directIn)")
tolog("Updated run command: %s" % cmd)
_runCommandList2.append(cmd)
_runCommandList = _runCommandList2
tolog("Dumping final input file states")
dumpFileStates(pworkdir, jobId, ftype="input")
return _runCommandList
def updateRunCommandList(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO):
""" update the run command list if --directIn is no longer needed """
# the method is using the file state dictionary
# remove later
dumpFileStates(pworkdir, jobId, type="input")
# are there only copy_to_scratch transfer modes in the file state dictionary?
# if so, remove any lingering --directIn instruction
only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
if only_copy_to_scratch:
# if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
_runCommandList = []
tolog("There are only copy_to_scratch transfer modes in file state dictionary")
for cmd in runCommandList:
# remove the --directIn string if present
if "--directIn" in cmd:
tolog("(Removing --directIn instruction from run command since it is not needed)")
cmd = cmd.replace("--directIn", "")
# remove the --useFileStager string if present
if "--useFileStager" in cmd:
tolog("(Removing --useFileStager instruction from run command since it is not needed)")
cmd = cmd.replace("--useFileStager", "")
# remove additional run options if creation of TURL based PFC failed
if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
if "--usePFCTurl" in cmd:
tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
cmd = cmd.replace(" --usePFCTurl", "")
if not "--lfcHost" in cmd and analysisJob:
tolog("Adding lfcHost to run command")
cmd += ' --lfcHost %s' % (readpar('lfchost'))
tolog("Updated run command: %s" % (cmd))
_runCommandList.append(cmd)
else:
tolog("Nothing to update in run command list related to copy-to-scratch")
_runCommandList = runCommandList
# was FAX used as primary site mover in combination with direct I/O?
if usedFAXandDirectIO:
tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
_runCommandList2 = []
for cmd in _runCommandList:
# remove the --lfcHost
if "--lfcHost" in cmd:
_lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
cmd = cmd.replace(_lfcHost, '')
tolog("(Removed the LFC host:%s)" % (_lfcHost))
# remove the --oldPrefix
if "--oldPrefix" in cmd:
pattern = "(\-\-oldPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --oldPrefix pattern)")
# remove the --newPrefix
if "--newPrefix" in cmd:
pattern = "(\-\-newPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --newPrefix pattern)")
# add the --usePFCTurl if not there already
if not "--usePFCTurl" in cmd:
cmd += " --usePFCTurl"
tolog("(Added --usePFCTurl)")
tolog("Updated run command: %s" % (cmd))
_runCommandList2.append(cmd)
_runCommandList = _runCommandList2
tolog("Dumping final input file states")
dumpFileStates(pworkdir, jobId, type="input")
return _runCommandList
def updateRunCommandListMJ(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO, hasInput):
"""!!! Should be merged with updateRunCommandList !!!"""
""" update the run command list if --directIn is no longer needed """
# the method is using the file state dictionary
# remove later
dumpFileStates(pworkdir, jobId, type="input")
# remove any instruction regarding tag file creation for event service jobs
#_runCommandList = []
# for cmd in runCommandList:
cmd = runCommandList['parameters']
if "--createTAGFileForES" in cmd:
cmd = cmd.replace("--createTAGFileForES","")
runCommandList['parameters'] = cmd
# no need to continue if no input files
if not hasInput:
return runCommandList
# are there only copy_to_scratch transfer modes in the file state dictionary?
# if so, remove any lingering --directIn instruction
only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
if only_copy_to_scratch:
# if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
_runCommandList = []
tolog("There are only copy_to_scratch transfer modes in file state dictionary")
#for cmd in runCommandList:
# remove the --directIn string if present
cmd = runCommandList['parameters']
if "--directIn" in cmd:
tolog("(Removing --directIn instruction from run command since it is not needed)")
cmd = cmd.replace("--directIn", "")
# remove the --useFileStager string if present
if "--useFileStager" in cmd:
tolog("(Removing --useFileStager instruction from run command since it is not needed)")
cmd = cmd.replace("--useFileStager", "")
# remove additional run options if creation of TURL based PFC failed
if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
if "--usePFCTurl" in cmd:
tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
cmd = cmd.replace(" --usePFCTurl", "")
if not "--lfcHost" in cmd and analysisJob:
tolog("Adding lfcHost to run command")
cmd += ' --lfcHost %s' % (readpar('lfchost'))
tolog("Updated run command: %s" % (cmd))
runCommandList['parameters'] = cmd
else:
tolog("Nothing to update in run command list related to copy-to-scratch")
# was FAX used as primary site mover in combination with direct I/O?
if usedFAXandDirectIO:
tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
# remove the --lfcHost
if "--lfcHost" in cmd:
_lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
cmd = cmd.replace(_lfcHost, '')
tolog("(Removed the LFC host:%s)" % (_lfcHost))
# remove the --oldPrefix
if "--oldPrefix" in cmd:
pattern = "(\-\-oldPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --oldPrefix pattern)")
# remove the --newPrefix
if "--newPrefix" in cmd:
pattern = "(\-\-newPrefix\ \S+)"
cmd = removePattern(cmd, pattern)
tolog("(Removed --newPrefix pattern)")
# add the --usePFCTurl if not there already
if not "--usePFCTurl" in cmd:
cmd += " --usePFCTurl"
tolog("(Added --usePFCTurl)")
tolog("Updated run command: %s" % (cmd))
runCommandList['parameters'] = cmd
tolog("Dumping final input file states")
dumpFileStates(pworkdir, jobId, type="input")
return runCommandList