예제 #1
0
def updateRunCommandList(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO, hasInput, prodDBlockToken):
    """ update the run command list if --directIn is no longer needed """
    # the method is using the file state dictionary

    # remove later
    dumpFileStates(pworkdir, jobId, ftype="input")

    # remove any instruction regarding tag file creation for event service jobs
    _runCommandList = []
    for cmd in runCommandList:
        if "--createTAGFileForES" in cmd:
            cmd = cmd.replace("--createTAGFileForES","")
        _runCommandList.append(cmd)
    runCommandList = _runCommandList

    # no need to continue if no input files
    if not hasInput:
        return runCommandList

    # are there only copy_to_scratch transfer modes in the file state dictionary?
    # if so, remove any lingering --directIn instruction
    only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
    if only_copy_to_scratch or 'local' in prodDBlockToken:
#    if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
        _runCommandList = []

        if only_copy_to_scratch:
            tolog("There are only copy_to_scratch transfer modes in file state dictionary")
        for cmd in runCommandList:
            # remove the --directIn string if present
            if "--directIn" in cmd:
                tolog("(Removing --directIn instruction from run command since it is not needed)")
                cmd = cmd.replace("--directIn", "")
            # remove the --useFileStager string if present
            if "--useFileStager" in cmd:
                tolog("(Removing --useFileStager instruction from run command since it is not needed)")
                cmd = cmd.replace("--useFileStager", "")
            # remove additional run options if creation of TURL based PFC failed
            if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
                if "--usePFCTurl" in cmd:
                    tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
                    cmd = cmd.replace(" --usePFCTurl", "")
                if not "--lfcHost" in cmd and analysisJob:
                    tolog("Adding lfcHost to run command")
                    cmd += ' --lfcHost %s' % (readpar('lfchost'))

            tolog("Updated run command: %s" % (cmd))
            _runCommandList.append(cmd)
    else:
        tolog("Nothing to update in run command list related to copy-to-scratch")
        _runCommandList = runCommandList

    # was FAX used as primary site mover in combination with direct I/O?
    if usedFAXandDirectIO == True:
        tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
        _runCommandList2 = []

        for cmd in _runCommandList:
            # remove the --lfcHost
            if "--lfcHost" in cmd:
                _lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
                cmd = cmd.replace(_lfcHost, '')
                tolog("(Removed the LFC host:%s)" % (_lfcHost))

            # remove the --oldPrefix
            if "--oldPrefix" in cmd:
                pattern = "(\-\-oldPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --oldPrefix pattern)")

            # remove the --newPrefix
            if "--newPrefix" in cmd:
                pattern = "(\-\-newPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --newPrefix pattern)")

            # add the --usePFCTurl if not there already
            if not "--usePFCTurl" in cmd and analysisJob:
                cmd += " --usePFCTurl"
                tolog("(Added --usePFCTurl)")

            tolog("Updated run command: %s" % (cmd))
            _runCommandList2.append(cmd)

        _runCommandList = _runCommandList2


    ### new movers quick integration: reuse usedFAXandDirectIO variable with special meaning
    ### to avoid any LFC and prefixes lookups in transformation scripts
    ### since new movers already form proper pfn values
    ### proper workflow is required: to be reimplemented later
    if usedFAXandDirectIO == 'newmover' or usedFAXandDirectIO == 'newmover-directaccess':
        tolog("updateRunCommandList(): use new movers logic")
        tolog("updateRunCommandList(): remove to be deprecated options (--lfcHost, --oldPrefix, --newPrefix) from command list")
        tolog("updateRunCommandList(): force to set --usePFCTurl")
        tolog("updateRunCommandList(): check directaccess mode if need (--directIn)")
        tolog("current runCommandList=%s" % _runCommandList)

        _runCommandList2 = []

        for cmd in _runCommandList:

            # remove the --lfcHost, --oldPrefix, --newPrefix
            # add --usePFCTurl

            if "--lfcHost" in cmd:
                cmd = removePattern(cmd, "(\-\-lfcHost\ \S+)")
                tolog("(Removed the --lfcHost)")

            if "--oldPrefix" in cmd:
                pattern = "(\-\-oldPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --oldPrefix pattern)")

            if "--newPrefix" in cmd:
                pattern = "(\-\-newPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --newPrefix pattern)")

            if "--usePFCTurl" not in cmd and analysisJob:
                cmd += " --usePFCTurl"
                tolog("(Added --usePFCTurl)")

            # add --directIn if need
            if usedFAXandDirectIO == 'newmover-directaccess':
                if "--directIn" not in cmd and analysisJob:
                    cmd += " --directIn"
                    tolog("(Added --directIn)")

            tolog("Updated run command: %s" % cmd)

            _runCommandList2.append(cmd)

        _runCommandList = _runCommandList2

    tolog("Dumping final input file states")
    dumpFileStates(pworkdir, jobId, ftype="input")

    return _runCommandList
예제 #2
0
def updateRunCommandList(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO, hasInput, prodDBlockToken):
    """ update the run command list if --directIn is no longer needed """
    # the method is using the file state dictionary

    # remove later
    dumpFileStates(pworkdir, jobId, ftype="input")

    # remove any instruction regarding tag file creation for event service jobs
    _runCommandList = []
    for cmd in runCommandList:
        if "--createTAGFileForES" in cmd:
            cmd = cmd.replace("--createTAGFileForES","")
        _runCommandList.append(cmd)
    runCommandList = _runCommandList

    # no need to continue if no input files
    if not hasInput:
        return runCommandList

    # are there only copy_to_scratch transfer modes in the file state dictionary?
    # if so, remove any lingering --directIn instruction
    only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
    if only_copy_to_scratch or 'local' in prodDBlockToken:
#    if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
        _runCommandList = []

        if only_copy_to_scratch:
            tolog("There are only copy_to_scratch transfer modes in file state dictionary")
        for cmd in runCommandList:
            # remove the --directIn string if present
            if "--directIn" in cmd:
                tolog("(Removing --directIn instruction from run command since it is not needed)")
                cmd = cmd.replace("--directIn", "")
            # remove the --useFileStager string if present
            if "--useFileStager" in cmd:
                tolog("(Removing --useFileStager instruction from run command since it is not needed)")
                cmd = cmd.replace("--useFileStager", "")
            # remove additional run options if creation of TURL based PFC failed
            if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
                if "--usePFCTurl" in cmd:
                    tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
                    cmd = cmd.replace(" --usePFCTurl", "")
                if not "--lfcHost" in cmd and analysisJob:
                    tolog("Adding lfcHost to run command")
                    cmd += ' --lfcHost %s' % (readpar('lfchost'))

            tolog("Updated run command: %s" % (cmd))
            _runCommandList.append(cmd)
    else:
        tolog("Nothing to update in run command list related to copy-to-scratch")
        _runCommandList = runCommandList

    # was FAX used as primary site mover in combination with direct I/O?
    if usedFAXandDirectIO == True:
        tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
        _runCommandList2 = []

        for cmd in _runCommandList:
            # remove the --lfcHost
            if "--lfcHost" in cmd:
                _lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
                cmd = cmd.replace(_lfcHost, '')
                tolog("(Removed the LFC host:%s)" % (_lfcHost))

            # remove the --oldPrefix
            if "--oldPrefix" in cmd:
                pattern = "(\-\-oldPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --oldPrefix pattern)")

            # remove the --newPrefix
            if "--newPrefix" in cmd:
                pattern = "(\-\-newPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --newPrefix pattern)")

            # add the --usePFCTurl if not there already
            if not "--usePFCTurl" in cmd and analysisJob:
                cmd += " --usePFCTurl"
                tolog("(Added --usePFCTurl)")

            tolog("Updated run command: %s" % (cmd))
            _runCommandList2.append(cmd)

        _runCommandList = _runCommandList2


    ### new movers quick integration: reuse usedFAXandDirectIO variable with special meaning
    ### to avoid any LFC and prefixes lookups in transformation scripts
    ### since new movers already form proper pfn values
    ### proper workflow is required: to be reimplemented later
    if usedFAXandDirectIO == 'newmover' or usedFAXandDirectIO == 'newmover-directaccess':
        _runCommandList2 = []

        for cmd in _runCommandList:

            # remove the --lfcHost, --oldPrefix, --newPrefix
            # add --usePFCTurl

            if "--lfcHost" in cmd:
                cmd = removePattern(cmd, "(\-\-lfcHost\ \S+)")
                tolog("(Removed the --lfcHost)")

            if "--oldPrefix" in cmd:
                pattern = "(\-\-oldPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --oldPrefix pattern)")

            if "--newPrefix" in cmd:
                pattern = "(\-\-newPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --newPrefix pattern)")

            if "--usePFCTurl" not in cmd and analysisJob:
                cmd += " --usePFCTurl"
                tolog("(Added --usePFCTurl)")

            # add --directIn if need
            if usedFAXandDirectIO == 'newmover-directaccess':
                if "--directIn" not in cmd and analysisJob:
                    cmd += " --directIn"
                    tolog("(Added --directIn)")

            tolog("Updated run command: %s" % cmd)

            _runCommandList2.append(cmd)

        _runCommandList = _runCommandList2

    tolog("Dumping final input file states")
    dumpFileStates(pworkdir, jobId, ftype="input")

    return _runCommandList
예제 #3
0
def updateRunCommandList(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO):
    """ update the run command list if --directIn is no longer needed """
    # the method is using the file state dictionary

    # remove later
    dumpFileStates(pworkdir, jobId, type="input")

    # are there only copy_to_scratch transfer modes in the file state dictionary?
    # if so, remove any lingering --directIn instruction
    only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
    if only_copy_to_scratch:
#    if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
        _runCommandList = []

        tolog("There are only copy_to_scratch transfer modes in file state dictionary")
        for cmd in runCommandList:
            # remove the --directIn string if present
            if "--directIn" in cmd:
                tolog("(Removing --directIn instruction from run command since it is not needed)")
                cmd = cmd.replace("--directIn", "")
            # remove the --useFileStager string if present
            if "--useFileStager" in cmd:
                tolog("(Removing --useFileStager instruction from run command since it is not needed)")
                cmd = cmd.replace("--useFileStager", "")
            # remove additional run options if creation of TURL based PFC failed
            if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
                if "--usePFCTurl" in cmd:
                    tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
                    cmd = cmd.replace(" --usePFCTurl", "")
                if not "--lfcHost" in cmd and analysisJob:
                    tolog("Adding lfcHost to run command")
                    cmd += ' --lfcHost %s' % (readpar('lfchost'))

            tolog("Updated run command: %s" % (cmd))
            _runCommandList.append(cmd)
    else:
        tolog("Nothing to update in run command list related to copy-to-scratch")
        _runCommandList = runCommandList

    # was FAX used as primary site mover in combination with direct I/O?
    if usedFAXandDirectIO:
        tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
        _runCommandList2 = []

        for cmd in _runCommandList:
            # remove the --lfcHost
            if "--lfcHost" in cmd:
                _lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
                cmd = cmd.replace(_lfcHost, '')
                tolog("(Removed the LFC host:%s)" % (_lfcHost))

            # remove the --oldPrefix
            if "--oldPrefix" in cmd:
                pattern = "(\-\-oldPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --oldPrefix pattern)")

            # remove the --newPrefix
            if "--newPrefix" in cmd:
                pattern = "(\-\-newPrefix\ \S+)"
                cmd = removePattern(cmd, pattern)
                tolog("(Removed --newPrefix pattern)")

            # add the --usePFCTurl if not there already
            if not "--usePFCTurl" in cmd:
                cmd += " --usePFCTurl"
                tolog("(Added --usePFCTurl)")

            tolog("Updated run command: %s" % (cmd))
            _runCommandList2.append(cmd)

        _runCommandList = _runCommandList2
    tolog("Dumping final input file states")
    dumpFileStates(pworkdir, jobId, type="input")

    return _runCommandList
def updateRunCommandListMJ(runCommandList, pworkdir, jobId, statusPFCTurl, analysisJob, usedFAXandDirectIO, hasInput):
    """!!! Should be merged with updateRunCommandList !!!"""
    """ update the run command list if --directIn is no longer needed    """
    # the method is using the file state dictionary

    # remove later
    dumpFileStates(pworkdir, jobId, type="input")

    # remove any instruction regarding tag file creation for event service jobs
    #_runCommandList = []
    # for cmd in runCommandList:
    cmd = runCommandList['parameters']
    if "--createTAGFileForES" in cmd:
        cmd = cmd.replace("--createTAGFileForES","")
    runCommandList['parameters'] = cmd
    
    # no need to continue if no input files
    if not hasInput:
        return runCommandList

    # are there only copy_to_scratch transfer modes in the file state dictionary?
    # if so, remove any lingering --directIn instruction
    only_copy_to_scratch = hasOnlyCopyToScratch(pworkdir, jobId)
    if only_copy_to_scratch:
#    if hasOnlyCopyToScratch(pworkdir, jobId): # python bug? does not work, have to use previous two lines?
        _runCommandList = []

        tolog("There are only copy_to_scratch transfer modes in file state dictionary")
        #for cmd in runCommandList:
        # remove the --directIn string if present
        cmd = runCommandList['parameters']
        if "--directIn" in cmd:
            tolog("(Removing --directIn instruction from run command since it is not needed)")
            cmd = cmd.replace("--directIn", "")
        # remove the --useFileStager string if present
        if "--useFileStager" in cmd:
            tolog("(Removing --useFileStager instruction from run command since it is not needed)")
            cmd = cmd.replace("--useFileStager", "")
        # remove additional run options if creation of TURL based PFC failed
        if statusPFCTurl == False: # (note: can also be None, so do not use 'if not statusPFCTurl')
            if "--usePFCTurl" in cmd:
                tolog("(Removing --usePFCTurl instruction from run command since it is not needed)")
                cmd = cmd.replace(" --usePFCTurl", "")
            if not "--lfcHost" in cmd and analysisJob:
                tolog("Adding lfcHost to run command")
                cmd += ' --lfcHost %s' % (readpar('lfchost'))

            tolog("Updated run command: %s" % (cmd))
            runCommandList['parameters'] = cmd
    else:
        tolog("Nothing to update in run command list related to copy-to-scratch")

    # was FAX used as primary site mover in combination with direct I/O?
    if usedFAXandDirectIO:
        tolog("Since FAX was used as primary site mover in combination with direct I/O, the run command list need to be updated")
        # remove the --lfcHost
        if "--lfcHost" in cmd:
            _lfcHost = ' --lfcHost %s' % (readpar('lfchost'))
            cmd = cmd.replace(_lfcHost, '')
            tolog("(Removed the LFC host:%s)" % (_lfcHost))

        # remove the --oldPrefix
        if "--oldPrefix" in cmd:
            pattern = "(\-\-oldPrefix\ \S+)"
            cmd = removePattern(cmd, pattern)
            tolog("(Removed --oldPrefix pattern)")

        # remove the --newPrefix
        if "--newPrefix" in cmd:
            pattern = "(\-\-newPrefix\ \S+)"
            cmd = removePattern(cmd, pattern)
            tolog("(Removed --newPrefix pattern)")

        # add the --usePFCTurl if not there already
        if not "--usePFCTurl" in cmd:
            cmd += " --usePFCTurl"
            tolog("(Added --usePFCTurl)")

        tolog("Updated run command: %s" % (cmd))
        runCommandList['parameters'] = cmd

    tolog("Dumping final input file states")
    dumpFileStates(pworkdir, jobId, type="input")

    return runCommandList