예제 #1
0
def main():
    Script.registerSwitch("e", "extended", "Show extended info")

    Script.parseCommandLine(ignoreErrors=True)

    from DIRAC import exit as DIRACExit
    from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin

    diracAdmin = DiracAdmin()
    exitCode = 0
    errorList = []
    extendedInfo = False

    for unprocSw in Script.getUnprocessedSwitches():
        if unprocSw[0] in ("e", "extended"):
            extendedInfo = True

    if not extendedInfo:
        result = diracAdmin.csListHosts()
        for host in result["Value"]:
            print(" %s" % host)
    else:
        result = diracAdmin.csDescribeHosts()
        print(diracAdmin.pPrint.pformat(result["Value"]))

    for error in errorList:
        print("ERROR %s: %s" % error)

    DIRACExit(exitCode)
    def registerSwitchesAndParseCommandLine(self):
        """Register the default plus additional parameters and parse options.

        :param list options: list of three tuple for options to add to the script
        :param list flags:  list of three tuple for flags to add to the script
        :param str opName
        """
        for short, longOption, doc in self.options:
            Script.registerSwitch(short + ":" if short else "",
                                  longOption + "=", doc)
        for short, longOption, doc in self.flags:
            Script.registerSwitch(short, longOption, doc)
            self.switches[longOption] = False
        Script.parseCommandLine()
        if Script.getPositionalArgs():
            Script.showHelp(exitCode=1)

        for switch in Script.getUnprocessedSwitches():
            for short, longOption, doc in self.options:
                if switch[0] == short or switch[0].lower() == longOption.lower(
                ):
                    sLog.verbose("Found switch %r with value %r" %
                                 (longOption, switch[1]))
                    self.switches[longOption] = switch[1]
                    break
            for short, longOption, doc in self.flags:
                if switch[0] == short or switch[0].lower() == longOption.lower(
                ):
                    self.switches[longOption] = True
                    break

        self.checkSwitches()
        self.switches["DryRun"] = not self.switches.get("Execute", False)
        self.switches["SourceSE"] = self.switches.get("SourceSE",
                                                      "").split(",")
예제 #3
0
def parseSwitches():
    """
    Parses the arguments passed by the user
    """

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()
    if args:
        subLogger.error(
            "Found the following positional args '%s', but we only accept switches"
            % args)
        subLogger.error("Please, check documentation below")
        Script.showHelp(exitCode=1)

    switches = dict(Script.getUnprocessedSwitches())

    # Default values
    switches.setdefault("element", None)
    switches.setdefault("defaultStatus", "Banned")
    if not switches["element"] in ("all", "Site", "Resource", "Node", None):
        subLogger.error("Found %s as element switch" % switches["element"])
        subLogger.error("Please, check documentation below")
        Script.showHelp(exitCode=1)

    subLogger.debug("The switches used are:")
    map(subLogger.debug, switches.items())

    return switches
예제 #4
0
def main():
    Script.registerSwitch(
        "v:",
        "vo=",
        "Location of pilot version in CS /Operations/<vo>/Pilot/Version"
        " (default value specified in CS under /DIRAC/DefaultSetup)",
    )
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument("version: pilot version you want to update to")
    Script.parseCommandLine(ignoreErrors=False)

    # parseCommandLine show help when mandatory arguments are not specified or incorrect argument
    version = Script.getPositionalArgs(group=True)

    vo = None
    for switch in Script.getUnprocessedSwitches():
        if switch[0] == "v" or switch[0] == "vo":
            vo = switch[1]

    from DIRAC import S_OK, S_ERROR
    from DIRAC import gConfig, gLogger
    from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI

    def updatePilot(version, vo):
        """
        Update in the CS the pilot version used,
        If only one version present in CS it's overwritten.
        If two versions present, the new one is added and the last removed

        :param version: version vArBpC of pilot you want to use
        :param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
        """
        setup = vo
        if not vo:
            setup = gConfig.getValue("/DIRAC/DefaultSetup")
        if not setup:
            return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")

        pilotVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup, [])
        if not pilotVersion:
            return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup)

        pilotVersion.pop()
        pilotVersion.insert(0, version)
        api = CSAPI()
        api.setOption("Operations/%s/Pilot/Version" % setup, ", ".join(pilotVersion))
        result = api.commit()
        if not result["OK"]:
            gLogger.fatal("Could not commit new version of pilot!")
            return result

        newVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup)
        return S_OK("New version of pilot set to %s" % newVersion)

    result = updatePilot(version, vo)
    if not result["OK"]:
        gLogger.fatal(result["Message"])
        DIRAC.exit(1)
    gLogger.notice(result["Value"])
    DIRAC.exit(0)
예제 #5
0
def parseSwitchesAndPositionalArguments():
    """
    Parse switches and positional arguments given to the script
    """

    # Parse the command line and initialize DIRAC
    Script.parseCommandLine(ignoreErrors=False)

    # Get arguments
    allArgs = Script.getPositionalArgs()
    gLogger.debug("All arguments: %s" % ", ".join(allArgs))

    # Get unprocessed switches
    switches = dict(Script.getUnprocessedSwitches())

    gLogger.debug("The switches used are:")
    map(gLogger.debug, switches.iteritems())

    # Get grouped positional arguments
    repType, user, services = Script.getPositionalArgs(group=True)
    gLogger.debug("The positional arguments are:")
    gLogger.debug("Report type:", repType)
    gLogger.debug("Name or DN:", user)
    gLogger.debug("Services:", services)

    return switches, repType, user, services
예제 #6
0
def processScriptSwitches():

    global vo, dry, doCEs, hostURL, onecore

    Script.registerSwitch("V:", "vo=", "Virtual Organization")
    Script.registerSwitch("D", "dry", "Dry run")
    Script.registerSwitch("C", "ce", "Process Computing Elements")
    Script.registerSwitch("H:", "host=", "use this url for information querying")
    Script.registerSwitch(
        "", "onecore", "Add Single Core Queues for each MultiCore Queue, set RequiredTag for those Queues"
    )
    Script.parseCommandLine(ignoreErrors=True)

    vo = ""
    dry = False
    doCEs = False
    hostURL = None
    onecore = False

    for sw in Script.getUnprocessedSwitches():
        if sw[0] in ("V", "vo"):
            vo = sw[1]
        if sw[0] in ("D", "dry"):
            dry = True
        if sw[0] in ("C", "ce"):
            doCEs = True
        if sw[0] in ("H", "host"):
            hostURL = sw[1]
        if sw[0] in ("onecore",):
            onecore = True
예제 #7
0
def main():
    Script.registerSwitch("", "Full", "   Print full list of requests")
    Script.parseCommandLine()
    from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient

    fullPrint = False

    for switch in Script.getUnprocessedSwitches():
        if switch[0] == "Full":
            fullPrint = True

    reqClient = ReqClient()

    for server, rpcClient in reqClient.requestProxies().items():
        DIRAC.gLogger.always("Checking request cache at %s" % server)
        reqCache = rpcClient.listCacheDir()
        if not reqCache["OK"]:
            DIRAC.gLogger.error("Cannot list request cache", reqCache)
            continue
        reqCache = reqCache["Value"]

        if not reqCache:
            DIRAC.gLogger.always("No request in cache")
        else:
            if fullPrint:
                DIRAC.gLogger.always("List of requests", reqCache)
            else:
                DIRAC.gLogger.always("Number of requests in the cache",
                                     len(reqCache))

    DIRAC.exit(0)
    def registerSwitchesAndParseCommandLine(self):
        """Register the default plus additional parameters and parse options.

        :param list options: list of three tuple for options to add to the script
        :param list flags:  list of three tuple for flags to add to the script
        :param str opName
        """
        for short, longOption, doc in self.options:
            Script.registerSwitch(short + ":" if short else "",
                                  longOption + "=", doc)
        for short, longOption, doc in self.flags:
            Script.registerSwitch(short, longOption, doc)
            self.switches[longOption] = False
        Script.parseCommandLine()
        if Script.getPositionalArgs():
            Script.showHelp(exitCode=1)

        ops = Operations()
        if not ops.getValue("DataManagement/ArchiveFiles/Enabled", False):
            sLog.error(
                'The "ArchiveFiles" operation is not enabled, contact your administrator!'
            )
            DIRAC.exit(1)
        for _short, longOption, _doc in self.options:
            defaultValue = ops.getValue(
                "DataManagement/ArchiveFiles/%s" % longOption, None)
            if defaultValue:
                sLog.verbose(
                    "Found default value in the CS for %r with value %r" %
                    (longOption, defaultValue))
                self.switches[longOption] = defaultValue
        for _short, longOption, _doc in self.flags:
            defaultValue = ops.getValue(
                "DataManagement/ArchiveFiles/%s" % longOption, False)
            if defaultValue:
                sLog.verbose(
                    "Found default value in the CS for %r with value %r" %
                    (longOption, defaultValue))
                self.switches[longOption] = defaultValue

        for switch in Script.getUnprocessedSwitches():
            for short, longOption, doc in self.options:
                if switch[0] == short or switch[0].lower() == longOption.lower(
                ):
                    sLog.verbose("Found switch %r with value %r" %
                                 (longOption, switch[1]))
                    self.switches[longOption] = switch[1]
                    break
            for short, longOption, doc in self.flags:
                if switch[0] == short or switch[0].lower() == longOption.lower(
                ):
                    self.switches[longOption] = True
                    break

        self.checkSwitches()
        self.switches["DryRun"] = not self.switches.get("Execute", False)
        self.switches["SourceSE"] = self.switches.get("SourceSE",
                                                      "").split(",")
def main():
    Script.registerSwitch("N:", "NumberOfProcessors=", "Run n parallel copies of the benchmark")
    Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value")
    Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value")
    Script.parseCommandLine(ignoreErrors=True)

    update = False
    configFile = None
    numberOfProcessors = 0

    for unprocSw in Script.getUnprocessedSwitches():
        if unprocSw[0] in ("U", "Update"):
            update = True
        elif unprocSw[0] in ("R", "Reconfig"):
            configFile = unprocSw[1]
        elif unprocSw[0] in ("N", "NumberOfProcessors"):
            try:
                numberOfProcessors = int(unprocSw[1])
            except ValueError:
                gLogger.warn("Cannot make benchmark measurements: NumberOfProcessors is not a number")

    # if numberOfProcessors has not been provided, try to get it from the configuration
    if not numberOfProcessors:
        numberOfProcessors = gConfig.getValue("/Resources/Computing/CEDefaults/NumberOfProcessors", 1)

    gLogger.info("Computing benchmark measurements on", "%d processor(s)..." % numberOfProcessors)

    # we want to get the logs coming from db12
    gLogger.enableLogsFromExternalLibs()
    # multiprocessor allocations generally have a CPU Power lower than single core one.
    # in order to avoid having wrong estimations, we run multiple copies of the benchmark simultaneously
    result = multipleDiracBenchmark(numberOfProcessors)

    if result is None:
        gLogger.error("Cannot make benchmark measurements")
        DIRAC.exit(1)

    # we take a conservative approach and use the minimum value returned as the CPU Power
    db12Result = min(result["raw"])
    # because hardware is continuously evolving, original benchmark scores might need a correction
    corr = Operations().getValue("JobScheduling/CPUNormalizationCorrection", 1.0)

    gLogger.info("Applying a correction on the CPU power:", corr)
    cpuPower = round(db12Result / corr, 1)

    gLogger.notice("Estimated CPU power is %.1f HS06" % cpuPower)

    if update:
        gConfig.setOptionValue("/LocalSite/CPUNormalizationFactor", cpuPower)

        if configFile:
            gConfig.dumpLocalCFGToFile(configFile)
        else:
            gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath)

    DIRAC.exit()
예제 #10
0
def main():
    Script.registerSwitch("p:", "property=",
                          "Add property to the user <name>=<value>")
    Script.registerSwitch("f", "force", "create the user if it doesn't exist")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(" user:     User name")
    Script.registerArgument(" DN:       DN of the User")
    Script.registerArgument(["group:    Add the user to the group"])
    Script.parseCommandLine(ignoreErrors=True)

    from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin

    diracAdmin = DiracAdmin()
    exitCode = 0
    forceCreation = False
    errorList = []

    userProps = {}
    for unprocSw in Script.getUnprocessedSwitches():
        if unprocSw[0] in ("f", "force"):
            forceCreation = True
        elif unprocSw[0] in ("p", "property"):
            prop = unprocSw[1]
            pl = prop.split("=")
            if len(pl) < 2:
                errorList.append((
                    "in arguments",
                    "Property %s has to include a '=' to separate name from value"
                    % prop))
                exitCode = 255
            else:
                pName = pl[0]
                pValue = "=".join(pl[1:])
                print("Setting property %s to %s" % (pName, pValue))
                userProps[pName] = pValue

    userName, userProps["DN"], userProps["Groups"] = Script.getPositionalArgs(
        group=True)

    if not diracAdmin.csModifyUser(
            userName, userProps, createIfNonExistant=forceCreation):
        errorList.append(("modify user", "Cannot modify user %s" % userName))
        exitCode = 255
    else:
        result = diracAdmin.csCommitChanges()
        if not result["OK"]:
            errorList.append(("commit", result["Message"]))
            exitCode = 255

    for error in errorList:
        print("ERROR %s: %s" % error)

    DIRAC.exit(exitCode)
예제 #11
0
def main():
    host = None
    Script.registerSwitch("H:", "host=", "   Target host")
    Script.parseCommandLine(ignoreErrors=False)
    for switch in Script.getUnprocessedSwitches():
        if switch[0].lower() == "h" or switch[0].lower() == "host":
            host = switch[1]

    from DIRAC.FrameworkSystem.Client.SystemAdministratorClientCLI import SystemAdministratorClientCLI

    cli = SystemAdministratorClientCLI(host)
    cli.cmdloop()
예제 #12
0
def parseSwitches():
    """
    Parses the arguments passed by the user
    """

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()
    if len(args) < 3:
        error(
            "Missing all mandatory 'query', 'element', 'tableType' arguments")
    elif args[0].lower() not in ("select", "add", "modify", "delete"):
        error("Incorrect 'query' argument")
    elif args[1].lower() not in ("site", "resource", "component", "node"):
        error("Incorrect 'element' argument")
    elif args[2].lower() not in ("status", "log", "history"):
        error("Incorrect 'tableType' argument")
    else:
        query = args[0].lower()

    switches = dict(Script.getUnprocessedSwitches())

    # Default values
    switches.setdefault("name", None)
    switches.setdefault("statusType", None)
    switches.setdefault("status", None)
    switches.setdefault("elementType", None)
    switches.setdefault("reason", None)
    switches.setdefault("lastCheckTime", None)
    switches.setdefault("tokenOwner", None)
    switches.setdefault("VO", None)

    if "status" in switches and switches["status"] is not None:
        switches["status"] = switches["status"].title()
        if switches["status"] not in ("Active", "Probing", "Degraded",
                                      "Banned", "Unknown", "Error"):
            error("'%s' is an invalid argument for switch 'status'" %
                  switches["status"])

    # when it's a add/modify query and status/reason/statusType are not specified
    # then some specific defaults are set up
    if query == "add" or query == "modify":
        if "status" not in switches or switches["status"] is None:
            switches["status"] = "Unknown"
        if "reason" not in switches or switches["reason"] is None:
            switches["reason"] = "Unknown reason"
        if "statusType" not in switches or switches["statusType"] is None:
            switches["statusType"] = "all"

    subLogger.debug("The switches used are:")
    map(subLogger.debug, switches.items())

    return args, switches
예제 #13
0
def main():
    Script.registerSwitch("e", "extended", "Show extended info")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(["Group:    Only users from this group (default: all)"], default=["all"], mandatory=False)
    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs(group=True)

    import DIRAC
    from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin

    diracAdmin = DiracAdmin()
    exitCode = 0
    errorList = []
    extendedInfo = False

    for unprocSw in Script.getUnprocessedSwitches():
        if unprocSw[0] in ("e", "extended"):
            extendedInfo = True

    def printUsersInGroup(group=False):
        result = diracAdmin.csListUsers(group)
        if result["OK"]:
            if group:
                print("Users in group %s:" % group)
            else:
                print("All users registered:")
            for username in result["Value"]:
                print(" %s" % username)

    def describeUsersInGroup(group=False):
        result = diracAdmin.csListUsers(group)
        if result["OK"]:
            if group:
                print("Users in group %s:" % group)
            else:
                print("All users registered:")
            result = diracAdmin.csDescribeUsers(result["Value"])
            print(diracAdmin.pPrint.pformat(result["Value"]))

    for group in args:
        if "all" in args:
            group = False
        if not extendedInfo:
            printUsersInGroup(group)
        else:
            describeUsersInGroup(group)

    for error in errorList:
        print("ERROR %s: %s" % error)

    DIRAC.exit(exitCode)
예제 #14
0
def main():
    Script.registerSwitch("C:", "CPUNormalizationFactor=",
                          "CPUNormalizationFactor, in case it is known")
    Script.parseCommandLine(ignoreErrors=True)

    CPUNormalizationFactor = 0.0
    for unprocSw in Script.getUnprocessedSwitches():
        if unprocSw[0] in ("C", "CPUNormalizationFactor"):
            CPUNormalizationFactor = float(unprocSw[1])

    from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getCPUTime

    cpuTime = getCPUTime(CPUNormalizationFactor)
    # I hate this kind of output... PhC
    print("CPU time left determined as", cpuTime)
    DIRAC.exit(0)
예제 #15
0
def main():
    global listOfFailedFiles

    Script.registerSwitch("D", "sync", "Make target directory identical to source")
    Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(
        (
            "LFN:           Logical File Name (Path to directory)",
            "Path:          Local path to the file (Path to directory)",
        )
    )
    Script.registerArgument(
        (
            "Path:          Local path to the file (Path to directory)",
            "LFN:           Logical File Name (Path to directory)",
        )
    )
    Script.registerArgument(" SE:            DIRAC Storage Element", mandatory=False)
    Script.parseCommandLine(ignoreErrors=False)

    args = Script.getPositionalArgs()
    if len(args) > 3:
        Script.showHelp()

    sync = False
    parallel = 1
    for switch in Script.getUnprocessedSwitches():
        if switch[0].lower() == "s" or switch[0].lower() == "sync":
            sync = True
        if switch[0].lower() == "j" or switch[0].lower() == "parallel":
            parallel = int(switch[1])

    listOfFailedFiles = Manager().list()

    # This is the execution
    returnValue = run(args, sync, parallel)
    if listOfFailedFiles:
        gLogger.error("Some file operations failed:\n\t", "\n\t".join(listOfFailedFiles))
        DIRAC.exit(1)
    if not returnValue["OK"]:
        gLogger.fatal(returnValue["Message"])
        DIRAC.exit(1)
    else:
        gLogger.notice(returnValue["Value"])
        DIRAC.exit(0)
예제 #16
0
def main():
    Script.registerSwitch("t", "test", "Only test. Don't commit changes")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(
        "UserCfg:  Cfg FileName with Users as sections containing"
        "DN, Groups, and other properties as options")
    Script.parseCommandLine(ignoreErrors=True)

    args = Script.getExtraCLICFGFiles()

    from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin

    diracAdmin = DiracAdmin()
    exitCode = 0
    testOnly = False
    errorList = []

    for unprocSw in Script.getUnprocessedSwitches():
        if unprocSw[0] in ("t", "test"):
            testOnly = True

    try:
        usersCFG = CFG().loadFromFile(args[0])
    except Exception as e:
        errorList.append("file open",
                         "Can't parse file %s: %s" % (args[0], str(e)))
        errorCode = 1
    else:
        if not diracAdmin.csSyncUsersWithCFG(usersCFG):
            errorList.append(("modify users", "Cannot sync with %s" % args[0]))
            exitCode = 255

    if not exitCode and not testOnly:
        result = diracAdmin.csCommitChanges()
        if not result["OK"]:
            errorList.append(("commit", result["Message"]))
            exitCode = 255

    for error in errorList:
        print("ERROR %s: %s" % error)

    DIRAC.exit(exitCode)
예제 #17
0
def parseSwitches():
    """
    Parses the arguments passed by the user
    """

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()
    if args:
        subLogger.error(
            "Found the following positional args '%s', but we only accept switches"
            % args)
        subLogger.error("Please, check documentation below")
        Script.showHelp(exitCode=1)

    switches = dict(Script.getUnprocessedSwitches())
    switches.setdefault("statusType", None)
    switches.setdefault("VO", None)

    for key in ("element", "name", "status", "reason"):

        if key not in switches:
            subLogger.error("%s Switch missing" % key)
            subLogger.error("Please, check documentation below")
            Script.showHelp(exitCode=1)

    if not switches["element"] in ("Site", "Resource", "Node"):
        subLogger.error("Found %s as element switch" % switches["element"])
        subLogger.error("Please, check documentation below")
        Script.showHelp(exitCode=1)

    statuses = StateMachine.RSSMachine(None).getStates()

    if not switches["status"] in statuses:
        subLogger.error("Found %s as element switch" % switches["element"])
        subLogger.error("Please, check documentation below")
        Script.showHelp(exitCode=1)

    subLogger.debug("The switches used are:")
    map(subLogger.debug, switches.items())

    return switches
예제 #18
0
def main():
    Script.registerSwitch("D:", "Download=",
                          "Defines data acquisition as DownloadInputData")
    Script.registerSwitch("P:", "Protocol=",
                          "Defines data acquisition as InputDataByProtocol")
    Script.parseCommandLine(ignoreErrors=False)

    _downloadinputdata = False
    _jobID = None

    for switch in Script.getUnprocessedSwitches():
        if switch[0] in ("D", "Download"):
            _downloadinputdata = True
            _jobID = switch[1]
        if switch[0] in ("I", "Protocol"):
            _downloadinputdata = False
            _jobID = switch[1]

    from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import Extensions

    ext = Extensions()
    _vo = ext.getCSExtensions()[0]
    _diracPath = Extensions().getExtensionPath("DIRAC")
    _dir = os.path.expanduser("~") + os.path.sep
    try:
        _path = __runSystemDefaults(_jobID, _vo)

        __downloadJobDescriptionXML(_jobID, _path)

        __modifyJobDescription(_jobID, _path, _downloadinputdata)

        __downloadPilotScripts(_path, _diracPath)

        __configurePilot(_path, _vo)

        __runJobLocally(_jobID, _path, _vo)

    finally:
        os.chdir(_dir)
        os.rename(_dir + ".dirac.cfg.old", _dir + ".dirac.cfg")
    def parseSwitches():
        """
        Parses the arguments passed by the user
        """

        Script.parseCommandLine(ignoreErrors=True)
        args = Script.getPositionalArgs()
        if args:
            subLogger.error(
                "Found the following positional args '%s', but we only accept switches"
                % args)
            subLogger.error("Please, check documentation below")
            Script.showHelp(exitCode=1)

        switches = dict(Script.getUnprocessedSwitches())

        for key in ("status", "se", "limit"):
            if key not in switches:
                subLogger.warn(
                    "You're not using switch --%s, query may take long!" % key)

        if "status" in switches and switches["status"] not in (
                "New",
                "Offline",
                "Waiting",
                "Failed",
                "StageSubmitted",
                "Staged",
        ):
            subLogger.error(
                'Found "%s" as Status value. Incorrect value used!' %
                switches["status"])
            subLogger.error("Please, check documentation below")
            Script.showHelp(exitCode=1)

        subLogger.debug("The switches used are:")
        map(subLogger.debug, switches.items())

        return switches
예제 #20
0
def parseSwitches():
    """
    Parses the arguments passed by the user
    """

    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()
    if not args:
        error("Missing mandatory 'query' argument")
    elif not args[0].lower() in ("select", "add", "delete"):
        error("Missing mandatory argument")
    else:
        query = args[0].lower()

    switches = dict(Script.getUnprocessedSwitches())

    # Default values
    switches.setdefault("downtimeID", None)
    switches.setdefault("element", None)
    switches.setdefault("name", None)
    switches.setdefault("startDate", None)
    switches.setdefault("endDate", None)
    switches.setdefault("severity", None)
    switches.setdefault("description", None)
    switches.setdefault("link", None)

    if query in ("add", "delete") and switches["downtimeID"] is None:
        error("'downtimeID' switch is mandatory for '%s' but found missing" % query)

    if query in ("add", "delete") and "ongoing" in switches:
        error("'ongoing' switch can be used only with 'select'")

    subLogger.debug("The switches used are:")
    map(subLogger.debug, switches.items())

    return (args, switches)
예제 #21
0
def main():
    site = "BOINC.World.org"
    status = ["Running"]
    minorStatus = None
    workerNodes = None
    since = None
    date = "today"
    full = False
    until = None
    batchIDs = None
    Script.registerSwitch("", "Site=", "   Select site (default: %s)" % site)
    Script.registerSwitch("", "Status=",
                          "   Select status (default: %s)" % status)
    Script.registerSwitch("", "MinorStatus=", "   Select minor status")
    Script.registerSwitch("", "WorkerNode=", "  Select WN")
    Script.registerSwitch("", "BatchID=", "  Select batch jobID")
    Script.registerSwitch(
        "", "Since=",
        "   Date since when to select jobs, or number of days (default: today)"
    )
    Script.registerSwitch("", "Date=",
                          "   Specify the date (check for a full day)")
    Script.registerSwitch(
        "", "Full",
        "   Printout full list of job (default: False except if --WorkerNode)")

    Script.parseCommandLine()
    from DIRAC import gLogger
    from DIRAC.Interfaces.API.Dirac import Dirac
    from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient

    switches = Script.getUnprocessedSwitches()
    for switch in switches:
        if switch[0] == "Site":
            site = switch[1]
        elif switch[0] == "MinorStatus":
            minorStatus = switch[1]
        elif switch[0] == "Status":
            if switch[1].lower() == "all":
                status = [None]
            else:
                status = switch[1].split(",")
        elif switch[0] == "WorkerNode":
            workerNodes = switch[1].split(",")
        elif switch[0] == "BatchID":
            try:
                batchIDs = [int(id) for id in switch[1].split(",")]
            except Exception:
                gLogger.error("Invalid jobID", switch[1])
                DIRAC.exit(1)
        elif switch[0] == "Full":
            full = True
        elif switch[0] == "Date":
            since = switch[1].split()[0]
            until = str(
                datetime.datetime.strptime(since, "%Y-%m-%d") +
                datetime.timedelta(days=1)).split()[0]
        elif switch[0] == "Since":
            date = switch[1].lower()
            if date == "today":
                since = None
            elif date == "yesterday":
                since = 1
            elif date == "ever":
                since = 2 * 365
            elif date.isdigit():
                since = int(date)
                date += " days"
            else:
                since = date
            if isinstance(since, int):
                since = str(datetime.datetime.now() -
                            datetime.timedelta(days=since)).split()[0]

    if workerNodes or batchIDs:
        # status = [None]
        full = True

    monitoring = JobMonitoringClient()
    dirac = Dirac()

    # Get jobs according to selection
    jobs = set()
    for stat in status:
        res = dirac.selectJobs(site=site,
                               date=since,
                               status=stat,
                               minorStatus=minorStatus)
        if not res["OK"]:
            gLogger.error("Error selecting jobs", res["Message"])
            DIRAC.exit(1)
        allJobs = set(int(job) for job in res["Value"])
        if until:
            res = dirac.selectJobs(site=site, date=until, status=stat)
            if not res["OK"]:
                gLogger.error("Error selecting jobs", res["Message"])
                DIRAC.exit(1)
            allJobs -= set(int(job) for job in res["Value"])
        jobs.update(allJobs)
    if not jobs:
        gLogger.always("No jobs found...")
        DIRAC.exit(0)
    # res = monitoring.getJobsSummary( jobs )
    # print eval( res['Value'] )[jobs[0]]

    allJobs = set()
    result = {}
    wnJobs = {}
    gLogger.always("%d jobs found" % len(jobs))
    # Get host name
    for job in jobs:
        res = monitoring.getJobParameter(job, "HostName")
        node = res.get("Value", {}).get("HostName", "Unknown")
        res = monitoring.getJobParameter(job, "LocalJobID")
        batchID = res.get("Value", {}).get("LocalJobID", "Unknown")
        if workerNodes:
            if not [wn for wn in workerNodes if node.startswith(wn)]:
                continue
            allJobs.add(job)
        if batchIDs:
            if batchID not in batchIDs:
                continue
            allJobs.add(job)
        if full or status == [None]:
            allJobs.add(job)
        result.setdefault(job, {})["Status"] = status
        result[job]["Node"] = node
        result[job]["LocalJobID"] = batchID
        wnJobs[node] = wnJobs.setdefault(node, 0) + 1

    # If necessary get jobs' status
    statusCounters = {}
    if allJobs:
        allJobs = sorted(allJobs, reverse=True)
        res = monitoring.getJobsStates(allJobs)
        if not res["OK"]:
            gLogger.error("Error getting job parameter", res["Message"])
        else:
            jobStates = res["Value"]
            for job in allJobs:
                stat = (
                    jobStates.get(job, {}).get("Status", "Unknown") + "; " +
                    jobStates.get(job, {}).get("MinorStatus", "Unknown") +
                    "; " +
                    jobStates.get(job, {}).get("ApplicationStatus", "Unknown"))
                result[job]["Status"] = stat
                statusCounters[stat] = statusCounters.setdefault(stat, 0) + 1
    elif not workerNodes and not batchIDs:
        allJobs = sorted(jobs, reverse=True)

    # Print out result
    if workerNodes or batchIDs:
        gLogger.always("Found %d jobs at %s, WN %s (since %s):" %
                       (len(allJobs), site, workerNodes, date))
        if allJobs:
            gLogger.always("List of jobs:",
                           ",".join([str(job) for job in allJobs]))
    else:
        if status == [None]:
            gLogger.always("Found %d jobs at %s (since %s):" %
                           (len(allJobs), site, date))
            for stat in sorted(statusCounters):
                gLogger.always("%d jobs %s" % (statusCounters[stat], stat))
        else:
            gLogger.always("Found %d jobs %s at %s (since %s):" %
                           (len(allJobs), status, site, date))
        gLogger.always(
            "List of WNs:",
            ",".join([
                "%s (%d)" % (node, wnJobs[node]) for node in sorted(
                    wnJobs,
                    key=cmp_to_key(lambda n1, n2: (wnJobs[n2] - wnJobs[n1])))
            ]),
        )
    if full:
        if workerNodes or batchIDs:
            nodeJobs = {}
            for job in allJobs:
                status = result[job]["Status"]
                node = result[job]["Node"].split(".")[0]
                jobID = result[job].get("LocalJobID")
                nodeJobs.setdefault(node, []).append((jobID, job, status))
            if not workerNodes:
                workerNodes = sorted(nodeJobs)
            for node in workerNodes:
                for job in nodeJobs.get(node.split(".")[0], []):
                    gLogger.always("%s " % node + "(%s): %s - %s" % job)
        else:
            for job in allJobs:
                status = result[job]["Status"]
                node = result[job]["Node"]
                jobID = result[job].get("LocalJobID")
                gLogger.always("%s (%s): %s - %s" % (node, jobID, job, status))
예제 #22
0
def main():
    days = 0
    months = 0
    years = 0
    wildcard = None
    baseDir = ""
    emptyDirsFlag = False
    Script.registerSwitch("D:", "Days=",
                          "Match files older than number of days [%s]" % days)
    Script.registerSwitch(
        "M:", "Months=",
        "Match files older than number of months [%s]" % months)
    Script.registerSwitch(
        "Y:", "Years=", "Match files older than number of years [%s]" % years)
    Script.registerSwitch("w:", "Wildcard=",
                          "Wildcard for matching filenames [All]")
    Script.registerSwitch(
        "b:", "BaseDir=",
        "Base directory to begin search (default /[vo]/user/[initial]/[username])"
    )
    Script.registerSwitch("e", "EmptyDirs",
                          "Create a list of empty directories")

    Script.parseCommandLine(ignoreErrors=False)

    for switch in Script.getUnprocessedSwitches():
        if switch[0] == "D" or switch[0].lower() == "days":
            days = int(switch[1])
        if switch[0] == "M" or switch[0].lower() == "months":
            months = int(switch[1])
        if switch[0] == "Y" or switch[0].lower() == "years":
            years = int(switch[1])
        if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
            wildcard = "*" + switch[1]
        if switch[0].lower() == "b" or switch[0].lower() == "basedir":
            baseDir = switch[1]
        if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
            emptyDirsFlag = True

    import DIRAC
    from DIRAC import gLogger
    from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
    from DIRAC.Core.Security.ProxyInfo import getProxyInfo
    from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
    from datetime import datetime, timedelta
    import sys
    import os
    import time
    import fnmatch

    fc = FileCatalog()

    def isOlderThan(cTimeStruct, days):
        timeDelta = timedelta(days=days)
        maxCTime = datetime.utcnow() - timeDelta
        if cTimeStruct < maxCTime:
            return True
        return False

    withMetadata = False
    if days or months or years:
        withMetadata = True
    totalDays = 0
    if years:
        totalDays += 365 * years
    if months:
        totalDays += 30 * months
    if days:
        totalDays += days

    res = getProxyInfo(False, False)
    if not res["OK"]:
        gLogger.error("Failed to get client proxy information.",
                      res["Message"])
        DIRAC.exit(2)
    proxyInfo = res["Value"]
    if proxyInfo["secondsLeft"] == 0:
        gLogger.error("Proxy expired")
        DIRAC.exit(2)
    username = proxyInfo["username"]
    vo = ""
    if "group" in proxyInfo:
        vo = getVOForGroup(proxyInfo["group"])
    if not baseDir:
        if not vo:
            gLogger.error("Could not determine VO")
            Script.showHelp()
        baseDir = "/%s/user/%s/%s" % (vo, username[0], username)

    baseDir = baseDir.rstrip("/")

    gLogger.notice("Will search for files in %s%s" %
                   (baseDir, (" matching %s" % wildcard) if wildcard else ""))
    activeDirs = [baseDir]

    allFiles = []
    emptyDirs = []

    while len(activeDirs) > 0:
        currentDir = activeDirs.pop()
        res = fc.listDirectory(currentDir, withMetadata, timeout=360)
        if not res["OK"]:
            gLogger.error("Error retrieving directory contents",
                          "%s %s" % (currentDir, res["Message"]))
        elif currentDir in res["Value"]["Failed"]:
            gLogger.error(
                "Error retrieving directory contents",
                "%s %s" % (currentDir, res["Value"]["Failed"][currentDir]))
        else:
            dirContents = res["Value"]["Successful"][currentDir]
            subdirs = dirContents["SubDirs"]
            files = dirContents["Files"]
            if not subdirs and not files:
                emptyDirs.append(currentDir)
                gLogger.notice("%s: empty directory" % currentDir)
            else:
                for subdir in sorted(subdirs, reverse=True):
                    if (not withMetadata) or isOlderThan(
                            subdirs[subdir]["CreationDate"], totalDays):
                        activeDirs.append(subdir)
                for filename in sorted(files):
                    fileOK = False
                    if (not withMetadata) or isOlderThan(
                            files[filename]["MetaData"]["CreationDate"],
                            totalDays):
                        if wildcard is None or fnmatch.fnmatch(
                                filename, wildcard):
                            fileOK = True
                    if not fileOK:
                        files.pop(filename)
                allFiles += sorted(files)

                if len(files) or len(subdirs):
                    gLogger.notice(
                        "%s: %d files%s, %d sub-directories" %
                        (currentDir, len(files), " matching"
                         if withMetadata or wildcard else "", len(subdirs)))

    outputFileName = "%s.lfns" % baseDir.replace("/%s" % vo,
                                                 "%s" % vo).replace("/", "-")
    outputFile = open(outputFileName, "w")
    for lfn in sorted(allFiles):
        outputFile.write(lfn + "\n")
    outputFile.close()
    gLogger.notice("%d matched files have been put in %s" %
                   (len(allFiles), outputFileName))

    if emptyDirsFlag:
        outputFileName = "%s.emptydirs" % baseDir.replace(
            "/%s" % vo, "%s" % vo).replace("/", "-")
        outputFile = open(outputFileName, "w")
        for dir in sorted(emptyDirs):
            outputFile.write(dir + "\n")
        outputFile.close()
        gLogger.notice("%d empty directories have been put in %s" %
                       (len(emptyDirs), outputFileName))

    DIRAC.exit(0)
def main():
    Script.registerSwitch("", "Status=", "Primary status")
    Script.registerSwitch("", "MinorStatus=", "Secondary status")
    Script.registerSwitch("", "ApplicationStatus=", "Application status")
    Script.registerSwitch("", "Site=", "Execution site")
    Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)")
    Script.registerSwitch("", "JobGroup=",
                          "Select jobs for specified job group")
    Script.registerSwitch(
        "", "Date=",
        "Date in YYYY-MM-DD format, if not specified default is today")
    Script.registerSwitch("", "File=",
                          "File name,if not specified default is std.out ")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument("String: string to search for")
    _, args = Script.parseCommandLine(ignoreErrors=True)

    # Default values
    status = None
    minorStatus = None
    appStatus = None
    site = None
    owner = None
    jobGroup = None
    date = None
    filename = "std.out"

    if len(args) != 1:
        Script.showHelp()

    searchstring = str(args[0])

    for switch in Script.getUnprocessedSwitches():
        if switch[0].lower() == "status":
            status = switch[1]
        elif switch[0].lower() == "minorstatus":
            minorStatus = switch[1]
        elif switch[0].lower() == "applicationstatus":
            appStatus = switch[1]
        elif switch[0].lower() == "site":
            site = switch[1]
        elif switch[0].lower() == "owner":
            owner = switch[1]
        elif switch[0].lower() == "jobgroup":
            jobGroup = switch[1]
        elif switch[0].lower() == "date":
            date = switch[1]
        elif switch[0].lower() == "file":
            filename = switch[1]

    selDate = date
    if not date:
        selDate = "Today"

    from DIRAC.Interfaces.API.Dirac import Dirac

    dirac = Dirac()
    exitCode = 0
    errorList = []
    resultDict = {}

    result = dirac.selectJobs(
        status=status,
        minorStatus=minorStatus,
        applicationStatus=appStatus,
        site=site,
        owner=owner,
        jobGroup=jobGroup,
        date=date,
    )
    if result["OK"]:
        jobs = result["Value"]
    else:
        print("Error in selectJob", result["Message"])
        DIRAC.exit(2)

    for job in jobs:

        result = dirac.getOutputSandbox(job)
        if result["OK"]:
            if os.path.exists("%s" % job):

                lines = []
                try:
                    lines = open(os.path.join(job, filename)).readlines()
                except Exception as x:
                    errorList.append((job, x))
                for line in lines:
                    if line.count(searchstring):
                        resultDict[job] = line
                rmtree("%s" % (job))
        else:
            errorList.append((job, result["Message"]))
            exitCode = 2

    for result in resultDict.items():
        print(result)

    DIRAC.exit(exitCode)
예제 #24
0
Script.registerSwitch("S:", "setup=",
                      "set the software dist module to update.")
Script.registerSwitch("D:", "softwareDistModule=",
                      "set the software dist module to update.")

Script.parseCommandLine()
args = Script.getPositionalArgs()

from DIRAC import gConfig

cFile = ""
sMod = ""
vo = ""
setup = ""

for unprocSw in Script.getUnprocessedSwitches():
    if unprocSw[0] in ("F", "file"):
        cFile = unprocSw[1]
    if unprocSw[0] in ("V", "vo"):
        vo = unprocSw[1]
    if unprocSw[0] in ("D", "softwareDistModule"):
        sMod = unprocSw[1]
    if unprocSw[0] in ("S", "setup"):
        setup = unprocSw[1]

localCfg = CFG()
if cFile:
    localConfigFile = cFile
else:
    print("WORKSPACE: %s" % os.path.expandvars("$WORKSPACE"))
    if os.path.isfile(
예제 #25
0
def main():
    catalog = None
    Script.registerSwitch("C:", "Catalog=", "Catalog to use")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(" requestName:  a request name")
    Script.registerArgument(" LFNs:         single LFN or file with LFNs")
    Script.registerArgument(["targetSE:     target SE"])
    Script.parseCommandLine()
    for switch in Script.getUnprocessedSwitches():
        if switch[0] == "C" or switch[0].lower() == "catalog":
            catalog = switch[1]

    args = Script.getPositionalArgs()

    requestName = None
    targetSEs = None
    if len(args) < 3:
        Script.showHelp(exitCode=1)

    requestName = args[0]
    lfnList = getLFNList(args[1])
    targetSEs = list(
        set([se for targetSE in args[2:] for se in targetSE.split(",")]))

    gLogger.info("Will create request '%s' with 'ReplicateAndRegister' "
                 "operation using %s lfns and %s target SEs" %
                 (requestName, len(lfnList), len(targetSEs)))

    from DIRAC.RequestManagementSystem.Client.Request import Request
    from DIRAC.RequestManagementSystem.Client.Operation import Operation
    from DIRAC.RequestManagementSystem.Client.File import File
    from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
    from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
    from DIRAC.Core.Utilities.List import breakListIntoChunks

    lfnChunks = breakListIntoChunks(lfnList, 100)
    multiRequests = len(lfnChunks) > 1

    error = 0
    count = 0
    reqClient = ReqClient()
    fc = FileCatalog()
    requestIDs = []
    for lfnChunk in lfnChunks:
        metaDatas = fc.getFileMetadata(lfnChunk)
        if not metaDatas["OK"]:
            gLogger.error("unable to read metadata for lfns: %s" %
                          metaDatas["Message"])
            error = -1
            continue
        metaDatas = metaDatas["Value"]
        for failedLFN, reason in metaDatas["Failed"].items():
            gLogger.error("skipping %s: %s" % (failedLFN, reason))
        lfnChunk = set(metaDatas["Successful"])

        if not lfnChunk:
            gLogger.error("LFN list is empty!!!")
            error = -1
            continue

        if len(lfnChunk) > Operation.MAX_FILES:
            gLogger.error(
                "too many LFNs, max number of files per operation is %s" %
                Operation.MAX_FILES)
            error = -1
            continue

        count += 1
        request = Request()
        request.RequestName = requestName if not multiRequests else "%s_%d" % (
            requestName, count)

        replicateAndRegister = Operation()
        replicateAndRegister.Type = "ReplicateAndRegister"
        replicateAndRegister.TargetSE = ",".join(targetSEs)
        if catalog is not None:
            replicateAndRegister.Catalog = catalog

        for lfn in lfnChunk:
            metaDict = metaDatas["Successful"][lfn]
            opFile = File()
            opFile.LFN = lfn
            opFile.Size = metaDict["Size"]

            if "Checksum" in metaDict:
                # # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
                opFile.Checksum = metaDict["Checksum"]
                opFile.ChecksumType = "ADLER32"
            replicateAndRegister.addFile(opFile)

        request.addOperation(replicateAndRegister)

        putRequest = reqClient.putRequest(request)
        if not putRequest["OK"]:
            gLogger.error("unable to put request '%s': %s" %
                          (request.RequestName, putRequest["Message"]))
            error = -1
            continue
        requestIDs.append(str(putRequest["Value"]))
        if not multiRequests:
            gLogger.always(
                "Request '%s' has been put to ReqDB for execution." %
                request.RequestName)

    if multiRequests:
        gLogger.always(
            "%d requests have been put to ReqDB for execution, with name %s_<num>"
            % (count, requestName))
    if requestIDs:
        gLogger.always("RequestID(s): %s" % " ".join(requestIDs))
    gLogger.always(
        "You can monitor requests' status using command: 'dirac-rms-request <requestName/ID>'"
    )
    DIRAC.exit(error)
예제 #26
0
def main():
    """
    Main executive code
    """
    Script.registerSwitch("", "Job=", "   JobID[,jobID2,...]")
    Script.registerSwitch("", "Transformation=", "   transformation ID")
    Script.registerSwitch("", "Tasks=", "      Associated to --Transformation, list of taskIDs")
    Script.registerSwitch("", "Verbose", "   Print more information")
    Script.registerSwitch("", "Terse", "   Only print request status")
    Script.registerSwitch("", "Full", "   Print full request content")
    Script.registerSwitch("", "Status=", "   Select all requests in a given status")
    Script.registerSwitch(
        "", "Since=", "      Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day"
    )
    Script.registerSwitch("", "Until=", "      Associated to --Status, end date (default= now")
    Script.registerSwitch("", "Maximum=", "      Associated to --Status, max number of requests ")
    Script.registerSwitch("", "Reset", "   Reset Failed files to Waiting if any")
    Script.registerSwitch("", "Force", "   Force reset even if not Failed")
    Script.registerSwitch(
        "", "All", "      (if --Status Failed) all requests, otherwise exclude irrecoverable failures"
    )
    Script.registerSwitch("", "FixJob", "   Set job Done if the request is Done")
    Script.registerSwitch("", "Cancel", "   Cancel the request")
    Script.registerSwitch("", "ListJobs", " List the corresponding jobs")
    Script.registerSwitch("", "TargetSE=", " Select request only if that SE is in the targetSEs")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(
        (
            "file:     a file containing a list of requests (Comma-separated on each line)",
            "request:  a request ID or a unique request name",
        ),
        mandatory=False,
    )
    Script.registerArgument(["request:  a request ID or a unique request name"], mandatory=False)
    Script.parseCommandLine()

    import DIRAC
    from DIRAC import gLogger

    jobs = []
    requestID = 0
    transID = None
    taskIDs = None
    tasks = None
    requests = []
    full = False
    verbose = False
    status = None
    until = None
    since = None
    terse = False
    allR = False
    reset = False
    fixJob = False
    maxRequests = 999999999999
    cancel = False
    listJobs = False
    force = False
    targetSE = set()
    for switch in Script.getUnprocessedSwitches():
        if switch[0] == "Job":
            jobs = []
            job = "Unknown"
            try:
                for arg in switch[1].split(","):
                    if os.path.exists(arg):
                        with open(arg, "r") as fp:
                            lines = fp.readlines()
                        for line in lines:
                            for job in line.split(","):
                                jobs += [int(job.strip())]
                        gLogger.notice("Found %d jobs in file %s" % (len(jobs), arg))
                    else:
                        jobs.append(int(arg))
            except TypeError:
                gLogger.fatal("Invalid jobID", job)
        elif switch[0] == "Transformation":
            try:
                transID = int(switch[1])
            except Exception:
                gLogger.fatal("Invalid transID", switch[1])
        elif switch[0] == "Tasks":
            try:
                taskIDs = [int(task) for task in switch[1].split(",")]
            except Exception:
                gLogger.fatal("Invalid tasks", switch[1])
        elif switch[0] == "Full":
            full = True
        elif switch[0] == "Verbose":
            verbose = True
        elif switch[0] == "Terse":
            terse = True
        elif switch[0] == "All":
            allR = True
        elif switch[0] == "Reset":
            reset = True
        elif switch[0] == "Force":
            force = True
        elif switch[0] == "Status":
            status = switch[1].capitalize()
        elif switch[0] == "Since":
            since = convertDate(switch[1])
        elif switch[0] == "Until":
            until = convertDate(switch[1])
        elif switch[0] == "FixJob":
            fixJob = True
        elif switch[0] == "Cancel":
            cancel = True
        elif switch[0] == "ListJobs":
            listJobs = True
        elif switch[0] == "Maximum":
            try:
                maxRequests = int(switch[1])
            except Exception:
                pass
        elif switch[0] == "TargetSE":
            targetSE = set(switch[1].split(","))

    if reset and not force:
        status = "Failed"
    if fixJob:
        status = "Done"
    if terse:
        verbose = True
    if status:
        if not until:
            until = datetime.datetime.utcnow()
        if not since:
            since = until - datetime.timedelta(hours=24)
    from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
    from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest

    reqClient = ReqClient()
    if transID:
        if not taskIDs:
            gLogger.fatal("If Transformation is set, a list of Tasks should also be set")
            Script.showHelp(exitCode=2)
        # In principle, the task name is unique, so the request name should be unique as well
        # If ever this would not work anymore, we would need to use the transformationClient
        # to fetch the ExternalID
        requests = ["%08d_%08d" % (transID, task) for task in taskIDs]
        allR = True

    elif not jobs:
        requests = []
        # Get full list of arguments, with and without comma
        for arg in [x.strip() for ar in Script.getPositionalArgs() for x in ar.split(",")]:
            if os.path.exists(arg):
                lines = open(arg, "r").readlines()
                requests += [reqID.strip() for line in lines for reqID in line.split(",")]
                gLogger.notice("Found %d requests in file" % len(requests))
            else:
                requests.append(arg)
            allR = True
    else:
        res = reqClient.getRequestIDsForJobs(jobs)
        if not res["OK"]:
            gLogger.fatal("Error getting request for jobs", res["Message"])
            DIRAC.exit(2)
        if res["Value"]["Failed"]:
            gLogger.error("No request found for jobs %s" % ",".join(sorted(str(job) for job in res["Value"]["Failed"])))
        requests = sorted(res["Value"]["Successful"].values())
        if requests:
            allR = True
        else:
            DIRAC.exit(0)

    if status and not requests:
        allR = allR or status != "Failed"
        res = reqClient.getRequestIDsList([status], limit=maxRequests, since=since, until=until)

        if not res["OK"]:
            gLogger.error("Error getting requests:", res["Message"])
            DIRAC.exit(2)
        requests = [reqID for reqID, _st, updTime in res["Value"] if updTime > since and updTime <= until and reqID]
        gLogger.notice("Obtained %d requests %s between %s and %s" % (len(requests), status, since, until))
    if not requests:
        gLogger.notice("No request selected....")
        Script.showHelp(exitCode=2)
    okRequests = []
    jobIDList = []
    for reqID in requests:
        # We allow reqID to be the requestName if it is unique
        try:
            # PEP-515 allows for underscore in numerical literals
            # So a request name 00123_00456
            # is interpreted as a requestID 12300456
            # Using an exception here for non-string is not an option
            if isinstance(reqID, six.string_types) and not reqID.isdigit():
                raise ValueError()

            requestID = int(reqID)
        except (ValueError, TypeError):
            requestID = reqClient.getRequestIDForName(reqID)
            if not requestID["OK"]:
                gLogger.notice(requestID["Message"])
                continue
            requestID = requestID["Value"]

        request = reqClient.peekRequest(requestID)
        if not request["OK"]:
            gLogger.error(request["Message"])
            DIRAC.exit(-1)

        request = request["Value"]
        if not request:
            gLogger.error("no such request %s" % requestID)
            continue
        # If no operation as the targetSE, skip
        if targetSE:
            found = False
            for op in request:
                if op.TargetSE and targetSE.intersection(op.TargetSE.split(",")):
                    found = True
                    break
            if not found:
                continue
        # keep a list of jobIDs if requested
        if request.JobID and listJobs:
            jobIDList.append(request.JobID)

        if status and request.Status != status:
            gLogger.notice(
                "Request %s is not in requested status %s%s" % (reqID, status, " (cannot be reset)" if reset else "")
            )
            continue

        if fixJob and request.Status == "Done" and request.JobID:
            # The request is for a job and is Done, verify that the job is in the proper status
            result = reqClient.finalizeRequest(request.RequestID, request.JobID, useCertificates=False)
            if not result["OK"]:
                gLogger.error("Error finalizing job", result["Message"])
            else:
                gLogger.notice("Job %d updated to %s" % (request.JobID, result["Value"]))
            continue

        if cancel:
            if request.Status not in ("Done", "Failed"):
                ret = reqClient.cancelRequest(requestID)
                if not ret["OK"]:
                    gLogger.error("Error canceling request %s" % reqID, ret["Message"])
                else:
                    gLogger.notice("Request %s cancelled" % reqID)
            else:
                gLogger.notice("Request %s is in status %s, not cancelled" % (reqID, request.Status))

        elif allR or recoverableRequest(request):
            okRequests.append(str(requestID))
            if reset:
                gLogger.notice("============ Request %s =============" % requestID)
                ret = reqClient.resetFailedRequest(requestID, allR=allR)
                if not ret["OK"]:
                    gLogger.error("Error resetting request %s" % requestID, ret["Message"])
            else:
                if len(requests) > 1:
                    gLogger.notice("\n===================================")
                dbStatus = reqClient.getRequestStatus(requestID).get("Value", "Unknown")
                printRequest(request, status=dbStatus, full=full, verbose=verbose, terse=terse)

    if listJobs:
        gLogger.notice("List of %d jobs:\n" % len(jobIDList), ",".join(str(jobID) for jobID in jobIDList))

    if status and okRequests:
        from DIRAC.Core.Utilities.List import breakListIntoChunks

        gLogger.notice("\nList of %d selected requests:" % len(okRequests))
        for reqs in breakListIntoChunks(okRequests, 100):
            gLogger.notice(",".join(reqs))
예제 #27
0
from DIRAC import gLogger
from DIRAC.Interfaces.API.Job import Job
from DIRAC.TransformationSystem.Client.Transformation import Transformation

# Needed to test transformations with Filters
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.DataManager import DataManager

# Parse the arguments
args = Script.getPositionalArgs()
if len(args) != 1:
    Script.showHelp()
directory = args[0]

UseFilter = None
for switch, switchValue in Script.getUnprocessedSwitches():
    if switch == "UseFilter":
        UseFilter = True if switchValue.lower() == "true" else False

# Let's first create the prodJobuction
prodJobType = "Merge"
transName = "testProduction_" + str(int(time.time()))
desc = "just test"

prodJob = Job()
prodJob._addParameter(prodJob.workflow, "PRODUCTION_ID", "string", "00012345",
                      "ProductionID")
prodJob._addParameter(prodJob.workflow, "JOB_ID", "string", "00006789",
                      "ProductionJobID")
prodJob._addParameter(prodJob.workflow, "eventType", "string", "TestEventType",
                      "Event Type of the prodJobuction")
예제 #28
0
def main():
    # Register workflow parameter switch
    Script.registerSwitch(
        "p:", "parameter=",
        "Parameters that are passed directly to the workflow")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(
        "jobXMLfile: specify path to the Job XML file description")
    Script.parseCommandLine()

    # from DIRAC.Core.Workflow.Parameter import *
    from DIRAC import gLogger
    from DIRAC.Core.Workflow.Workflow import fromXMLFile
    from DIRAC.Core.Utilities.Proxy import executeWithoutServerCertificate
    from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
    from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
    from DIRAC.RequestManagementSystem.Client.Request import Request

    # Forcing the current directory to be the first in the PYTHONPATH
    sys.path.insert(0, os.path.realpath("."))
    gLogger.showHeaders(True)

    @executeWithoutServerCertificate
    def jobexec(jobxml, wfParameters):
        jobfile = os.path.abspath(jobxml)
        if not os.path.exists(jobfile):
            gLogger.warn("Path to specified workflow %s does not exist" %
                         (jobfile))
            sys.exit(1)
        workflow = fromXMLFile(jobfile)
        gLogger.debug(workflow)
        code = workflow.createCode()
        gLogger.debug(code)
        jobID = 0
        if "JOBID" in os.environ:
            jobID = os.environ["JOBID"]
            gLogger.info("DIRAC JobID %s is running at site %s" %
                         (jobID, DIRAC.siteName()))

        workflow.addTool("JobReport", JobReport(jobID))
        workflow.addTool("AccountingReport", DataStoreClient())
        workflow.addTool("Request", Request())

        # Propagate the command line parameters to the workflow if any
        for pName, pValue in wfParameters.items():
            workflow.setValue(pName, pValue)

        # Propagate the command line parameters to the workflow module instances of each step
        for stepdefinition in workflow.step_definitions.values():
            for moduleInstance in stepdefinition.module_instances:
                for pName, pValue in wfParameters.items():
                    if moduleInstance.parameters.find(pName):
                        moduleInstance.parameters.setValue(pName, pValue)

        return workflow.execute()

    positionalArgs = Script.getPositionalArgs()
    if len(positionalArgs) != 1:
        gLogger.debug("Positional arguments were %s" % (positionalArgs))
        DIRAC.abort(1, "Must specify the Job XML file description")

    if "JOBID" in os.environ:
        gLogger.info("JobID: %s" % (os.environ["JOBID"]))

    jobXMLfile = positionalArgs[0]
    parList = Script.getUnprocessedSwitches()
    parDict = {}
    for switch, parameter in parList:
        if switch == "p":
            name, value = parameter.split("=")
            value = value.strip()

            # The comma separated list in curly brackets is interpreted as a list
            if value.startswith("{"):
                value = value[1:-1].replace('"', "").replace(" ",
                                                             "").split(",")
                value = ";".join(value)

            parDict[name] = value

    gLogger.debug("PYTHONPATH:\n%s" % ("\n".join(sys.path)))
    jobExec = jobexec(jobXMLfile, parDict)
    if not jobExec["OK"]:
        gLogger.debug("Workflow execution finished with errors, exiting")
        if jobExec["Errno"]:
            os._exit(jobExec["Errno"])
        else:
            os._exit(1)
    else:
        gLogger.debug("Workflow execution successful, exiting")
        # dirac_jobexec might interact with ARC library which cannot be closed using a simple sys.exit(0)
        # See https://bugzilla.nordugrid.org/show_bug.cgi?id=4022 for further details
        os._exit(0)
예제 #29
0
def main():
    Script.registerSwitch("E:", "email=",
                          "Boolean True/False (True by default)")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument("Site:     Name of the Site")
    Script.registerArgument("Comment:  Reason of the action")
    Script.parseCommandLine(ignoreErrors=True)

    from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
    from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
    from DIRAC import exit as DIRACExit, gConfig, gLogger

    def getBoolean(value):
        if value.lower() == "true":
            return True
        elif value.lower() == "false":
            return False
        else:
            Script.showHelp()

    email = True
    for switch in Script.getUnprocessedSwitches():
        if switch[0] == "email":
            email = getBoolean(switch[1])

    diracAdmin = DiracAdmin()
    exitCode = 0
    errorList = []
    setup = gConfig.getValue("/DIRAC/Setup", "")
    if not setup:
        print("ERROR: Could not contact Configuration Service")
        exitCode = 2
        DIRACExit(exitCode)

    # result = promptUser(
    #     'All the elements that are associated with this site will be active, '
    #     'are you sure about this action?'
    # )
    # if not result['OK'] or result['Value'] is 'n':
    #  print 'Script stopped'
    #  DIRACExit( 0 )

    # parseCommandLine show help when mandatory arguments are not specified or incorrect argument
    site, comment = Script.getPositionalArgs(group=True)
    result = diracAdmin.allowSite(site, comment, printOutput=True)
    if not result["OK"]:
        errorList.append((site, result["Message"]))
        exitCode = 2
    else:
        if email:
            userName = diracAdmin._getCurrentUser()
            if not userName["OK"]:
                print("ERROR: Could not obtain current username from proxy")
                exitCode = 2
                DIRACExit(exitCode)
            userName = userName["Value"]
            subject = "%s is added in site mask for %s setup" % (site, setup)
            body = "Site %s is added to the site mask for %s setup by %s on %s.\n\n" % (
                site,
                setup,
                userName,
                time.asctime(),
            )
            body += "Comment:\n%s" % comment
            addressPath = "EMail/Production"
            address = Operations().getValue(addressPath, "")
            if not address:
                gLogger.notice(
                    "'%s' not defined in Operations, can not send Mail\n" %
                    addressPath, body)
            else:
                result = diracAdmin.sendMail(address, subject, body)
        else:
            print("Automatic email disabled by flag.")

    for error in errorList:
        print("ERROR %s: %s" % error)

    DIRACExit(exitCode)
예제 #30
0
def main():
    Script.registerSwitch("", "Path=", "    Path to search for")
    Script.registerSwitch(
        "", "SE=",
        "    (comma-separated list of) SEs/SE-groups to be searched")
    # Registering arguments will automatically add their description to the help menu
    Script.registerArgument(
        [
            "metaspec: metadata index specification (of the form: "
            '"meta=value" or "meta<value", "meta!=value", etc.)'
        ],
        mandatory=False,
    )
    Script.parseCommandLine(ignoreErrors=True)
    args = Script.getPositionalArgs()

    import DIRAC
    from DIRAC import gLogger
    from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
    from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
    from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup

    path = "/"
    seList = None
    for opt, val in Script.getUnprocessedSwitches():
        if opt == "Path":
            path = val
        elif opt == "SE":
            seList = resolveSEGroup(val.split(","))

    if seList:
        args.append("SE=%s" % ",".join(seList))
    fc = FileCatalog()
    result = fc.getMetadataFields()
    if not result["OK"]:
        gLogger.error("Can not access File Catalog:", result["Message"])
        DIRAC.exit(-1)
    typeDict = result["Value"]["FileMetaFields"]
    typeDict.update(result["Value"]["DirectoryMetaFields"])
    # Special meta tags
    typeDict.update(FILE_STANDARD_METAKEYS)

    if len(args) < 1:
        print("Error: No argument provided\n%s:" % Script.scriptName)
        gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
        Script.showHelp(exitCode=1)

    mq = MetaQuery(typeDict=typeDict)
    result = mq.setMetaQuery(args)
    if not result["OK"]:
        gLogger.error("Illegal metaQuery:", result["Message"])
        DIRAC.exit(-1)
    metaDict = result["Value"]
    path = metaDict.pop("Path", path)

    result = fc.findFilesByMetadata(metaDict, path)
    if not result["OK"]:
        gLogger.error("Can not access File Catalog:", result["Message"])
        DIRAC.exit(-1)
    lfnList = sorted(result["Value"])

    gLogger.notice("\n".join(lfn for lfn in lfnList))