def parseSwitchesAndPositionalArguments(): """ Parse switches and positional arguments given to the script """ # Parse the command line and initialize DIRAC Script.parseCommandLine(ignoreErrors=False) # Get arguments allArgs = Script.getPositionalArgs() gLogger.debug("All arguments: %s" % ", ".join(allArgs)) # Get unprocessed switches switches = dict(Script.getUnprocessedSwitches()) gLogger.debug("The switches used are:") map(gLogger.debug, switches.iteritems()) # Get grouped positional arguments repType, user, services = Script.getPositionalArgs(group=True) gLogger.debug("The positional arguments are:") gLogger.debug("Report type:", repType) gLogger.debug("Name or DN:", user) gLogger.debug("Services:", services) return switches, repType, user, services
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["LFN: Logical File Name or file containing LFNs"]) Script.parseCommandLine(ignoreErrors=True) lfns = Script.getPositionalArgs() if len(lfns) < 1: Script.showHelp() from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 if len(lfns) == 1: try: with open(lfns[0], "r") as f: lfns = f.read().splitlines() except Exception: pass result = dirac.getFile(lfns, printOutput=True) if not result["OK"]: print("ERROR %s" % (result["Message"])) exitCode = 2 DIRAC.exit(exitCode)
def main(): Script.registerSwitch( "v:", "vo=", "Location of pilot version in CS /Operations/<vo>/Pilot/Version" " (default value specified in CS under /DIRAC/DefaultSetup)", ) # Registering arguments will automatically add their description to the help menu Script.registerArgument("version: pilot version you want to update to") Script.parseCommandLine(ignoreErrors=False) # parseCommandLine show help when mandatory arguments are not specified or incorrect argument version = Script.getPositionalArgs(group=True) vo = None for switch in Script.getUnprocessedSwitches(): if switch[0] == "v" or switch[0] == "vo": vo = switch[1] from DIRAC import S_OK, S_ERROR from DIRAC import gConfig, gLogger from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI def updatePilot(version, vo): """ Update in the CS the pilot version used, If only one version present in CS it's overwritten. If two versions present, the new one is added and the last removed :param version: version vArBpC of pilot you want to use :param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version """ setup = vo if not vo: setup = gConfig.getValue("/DIRAC/DefaultSetup") if not setup: return S_ERROR("No value set for /DIRAC/DefaultSetup in CS") pilotVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup, []) if not pilotVersion: return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup) pilotVersion.pop() pilotVersion.insert(0, version) api = CSAPI() api.setOption("Operations/%s/Pilot/Version" % setup, ", ".join(pilotVersion)) result = api.commit() if not result["OK"]: gLogger.fatal("Could not commit new version of pilot!") return result newVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup) return S_OK("New version of pilot set to %s" % newVersion) result = updatePilot(version, vo) if not result["OK"]: gLogger.fatal(result["Message"]) DIRAC.exit(1) gLogger.notice(result["Value"]) DIRAC.exit(0)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument( ("LFN: LFN", "File: File name containing a list of affected LFNs")) Script.registerArgument(" SE: Name of Storage Element") Script.registerArgument(" Status: New Status for the replica") Script.parseCommandLine(ignoreErrors=False) import DIRAC from DIRAC import gLogger from DIRAC.Resources.Catalog.FileCatalog import FileCatalog import os # parseCommandLine show help when mandatory arguments are not specified or incorrect argument inputFileName, storageElement, status = Script.getPositionalArgs( group=True) if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() inputFile.close() lfns = sorted(string.splitlines()) else: lfns = [inputFileName] fc = FileCatalog() res = fc.getReplicas(lfns, allStatus=True) if not res["OK"]: gLogger.error("Failed to get catalog replicas.", res["Message"]) DIRAC.exit(-1) lfnDict = {} for lfn, error in res["Value"]["Failed"].items(): gLogger.error("Failed to get replicas for file.", "%s:%s" % (lfn, error)) for lfn, replicas in res["Value"]["Successful"].items(): if storageElement not in replicas.keys(): gLogger.error("LFN not registered at provided storage element.", "%s %s" % (lfn, storageElement)) else: lfnDict[lfn] = { "SE": storageElement, "PFN": replicas[storageElement], "Status": status } if not lfnDict: gLogger.error("No files found at the supplied storage element.") DIRAC.exit(2) res = fc.setReplicaStatus(lfnDict) if not res["OK"]: gLogger.error("Failed to set catalog replica status.", res["Message"]) DIRAC.exit(-1) for lfn, error in res["Value"]["Failed"].items(): gLogger.error("Failed to set replica status for file.", "%s:%s" % (lfn, error)) gLogger.notice("Successfully updated the status of %d files at %s." % (len(res["Value"]["Successful"].keys()), storageElement)) DIRAC.exit(0)
def main(): Script.registerArgument(["Agent: specify which agent to run"]) positionalArgs = Script.getPositionalArgs(group=True) localCfg = Script.localCfg agentName = positionalArgs[0] localCfg.setConfigurationForAgent(agentName) localCfg.addMandatoryEntry("/DIRAC/Setup") localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes") localCfg.addDefaultEntry("LogLevel", "INFO") localCfg.addDefaultEntry("LogColor", True) resultDict = localCfg.loadUserData() if not resultDict["OK"]: gLogger.error("There were errors when loading configuration", resultDict["Message"]) sys.exit(1) includeExtensionErrors() agentReactor = AgentReactor(positionalArgs[0]) result = agentReactor.loadAgentModules(positionalArgs) if result["OK"]: agentReactor.go() # dirac_agent might interact with ARC library which cannot be closed using a simple sys.exit(0) # See https://bugzilla.nordugrid.org/show_bug.cgi?id=4022 for further details os._exit(0) else: gLogger.error("Error while loading agent module", result["Message"]) sys.exit(2)
def registerSwitchesAndParseCommandLine(self): """Register the default plus additional parameters and parse options. :param list options: list of three tuple for options to add to the script :param list flags: list of three tuple for flags to add to the script :param str opName """ for short, longOption, doc in self.options: Script.registerSwitch(short + ":" if short else "", longOption + "=", doc) for short, longOption, doc in self.flags: Script.registerSwitch(short, longOption, doc) self.switches[longOption] = False Script.parseCommandLine() if Script.getPositionalArgs(): Script.showHelp(exitCode=1) for switch in Script.getUnprocessedSwitches(): for short, longOption, doc in self.options: if switch[0] == short or switch[0].lower() == longOption.lower( ): sLog.verbose("Found switch %r with value %r" % (longOption, switch[1])) self.switches[longOption] = switch[1] break for short, longOption, doc in self.flags: if switch[0] == short or switch[0].lower() == longOption.lower( ): self.switches[longOption] = True break self.checkSwitches() self.switches["DryRun"] = not self.switches.get("Execute", False) self.switches["SourceSE"] = self.switches.get("SourceSE", "").split(",")
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) switches = dict(Script.getUnprocessedSwitches()) # Default values switches.setdefault("element", None) switches.setdefault("defaultStatus", "Banned") if not switches["element"] in ("all", "Site", "Resource", "Node", None): subLogger.error("Found %s as element switch" % switches["element"]) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") map(subLogger.debug, switches.items()) return switches
def registerSwitchesAndParseCommandLine(self): """Register the default plus additional parameters and parse options. :param list options: list of three tuple for options to add to the script :param list flags: list of three tuple for flags to add to the script :param str opName """ for short, longOption, doc in self.options: Script.registerSwitch(short + ":" if short else "", longOption + "=", doc) for short, longOption, doc in self.flags: Script.registerSwitch(short, longOption, doc) self.switches[longOption] = False Script.parseCommandLine() if Script.getPositionalArgs(): Script.showHelp(exitCode=1) ops = Operations() if not ops.getValue("DataManagement/ArchiveFiles/Enabled", False): sLog.error( 'The "ArchiveFiles" operation is not enabled, contact your administrator!' ) DIRAC.exit(1) for _short, longOption, _doc in self.options: defaultValue = ops.getValue( "DataManagement/ArchiveFiles/%s" % longOption, None) if defaultValue: sLog.verbose( "Found default value in the CS for %r with value %r" % (longOption, defaultValue)) self.switches[longOption] = defaultValue for _short, longOption, _doc in self.flags: defaultValue = ops.getValue( "DataManagement/ArchiveFiles/%s" % longOption, False) if defaultValue: sLog.verbose( "Found default value in the CS for %r with value %r" % (longOption, defaultValue)) self.switches[longOption] = defaultValue for switch in Script.getUnprocessedSwitches(): for short, longOption, doc in self.options: if switch[0] == short or switch[0].lower() == longOption.lower( ): sLog.verbose("Found switch %r with value %r" % (longOption, switch[1])) self.switches[longOption] = switch[1] break for short, longOption, doc in self.flags: if switch[0] == short or switch[0].lower() == longOption.lower( ): self.switches[longOption] = True break self.checkSwitches() self.switches["DryRun"] = not self.switches.get("Execute", False) self.switches["SourceSE"] = self.switches.get("SourceSE", "").split(",")
def main(): global overwrite Script.registerSwitch("f", "force", "Force overwrite of existing file", setOverwrite) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1 or len(args) > 4: Script.showHelp(exitCode=1) from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC import gLogger import DIRAC exitCode = 0 lfns = [] if len(args) == 1: inputFileName = args[0] if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") for line in inputFile: line = line.rstrip() items = line.split() items[0] = items[0].replace("LFN:", "").replace("lfn:", "") lfns.append(getDict(items)) inputFile.close() else: gLogger.error("Error: LFN list '%s' missing." % inputFileName) exitCode = 4 else: lfns.append(getDict(args)) dm = DataManager() for lfn in lfns: if not os.path.exists(lfn["localfile"]): gLogger.error("File %s must exist locally" % lfn["localfile"]) exitCode = 1 continue if not os.path.isfile(lfn["localfile"]): gLogger.error("%s is not a file" % lfn["localfile"]) exitCode = 2 continue gLogger.notice("\nUploading %s" % lfn["lfn"]) res = dm.putAndRegister(lfn["lfn"], lfn["localfile"], lfn["SE"], lfn["guid"], overwrite=overwrite) if not res["OK"]: exitCode = 3 gLogger.error("Error: failed to upload %s to %s: %s" % (lfn["lfn"], lfn["SE"], res)) continue else: gLogger.notice("Successfully uploaded file to %s" % lfn["SE"]) DIRAC.exit(exitCode)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument("prodID: Production ID") Script.registerArgument("transID: Transformation ID") Script.registerArgument("parentTransID: Parent Transformation ID", default="", mandatory=False) _, args = Script.parseCommandLine() from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient prodClient = ProductionClient() transClient = TransformationClient() # get arguments prodID, transID, parentTransID = Script.getPositionalArgs(group=True) if len(args) > 3: Script.showHelp(exitCode=1) res = transClient.getTransformation(transID) if not res["OK"]: DIRAC.gLogger.error("Failed to get transformation %s: %s" % (transID, res["Message"])) DIRAC.exit(-1) transID = res["Value"]["TransformationID"] if parentTransID: res = transClient.getTransformation(parentTransID) if not res["OK"]: DIRAC.gLogger.error("Failed to get transformation %s: %s" % (parentTransID, res["Message"])) DIRAC.exit(-1) parentTransID = res["Value"]["TransformationID"] res = prodClient.getProduction(prodID) if not res["OK"]: DIRAC.gLogger.error("Failed to get production %s: %s" % (prodID, res["Message"])) DIRAC.exit(-1) prodID = res["Value"]["ProductionID"] res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID) if not res["OK"]: DIRAC.gLogger.error(res["Message"]) DIRAC.exit(-1) if parentTransID: msg = "Transformation %s successfully added to production %s with parent transformation %s" % ( transID, prodID, parentTransID, ) else: msg = "Transformation %s successfully added to production %s with no parent transformation" % (transID, prodID) DIRAC.gLogger.notice(msg) DIRAC.exit(0)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument( "Request: ID of the Stage request in the StorageManager") Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if not len(args) == 1: Script.showHelp() from DIRAC import exit as DIRACExit, gLogger try: taskID = int(args[0]) except Exception: gLogger.fatal("Stage requestID must be an integer") DIRACExit(2) from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient client = StorageManagerClient() res = client.getTaskSummary(taskID) if not res["OK"]: gLogger.error(res["Message"]) DIRACExit(2) taskInfo = res["Value"]["TaskInfo"] replicaInfo = res["Value"]["ReplicaInfo"] outStr = "%s: %s" % ("TaskID".ljust(20), taskID) outStr += "\n%s: %s" % ("Status".ljust(20), taskInfo[taskID]["Status"]) outStr += "\n%s: %s" % ("Source".ljust(20), taskInfo[taskID]["Source"]) outStr += "\n%s: %s" % ("SourceTaskID".ljust(20), taskInfo[taskID]["SourceTaskID"]) outStr += "\n%s: %s" % ("CallBackMethod".ljust(20), taskInfo[taskID]["CallBackMethod"]) outStr += "\n%s: %s" % ("SubmitTime".ljust(20), taskInfo[taskID]["SubmitTime"]) outStr += "\n%s: %s" % ("CompleteTime".ljust(20), taskInfo[taskID]["CompleteTime"]) for lfn, metadata in replicaInfo.items(): outStr += "\n" outStr += "\n\t%s: %s" % ("LFN".ljust(8), lfn.ljust(100)) outStr += "\n\t%s: %s" % ("SE".ljust(8), metadata["StorageElement"].ljust(100)) outStr += "\n\t%s: %s" % ("PFN".ljust(8), str( metadata["PFN"]).ljust(100)) outStr += "\n\t%s: %s" % ("Size".ljust(8), str( metadata["FileSize"]).ljust(100)) outStr += "\n\t%s: %s" % ("Status".ljust(8), metadata["Status"].ljust(100)) outStr += "\n\t%s: %s" % ("Reason".ljust(8), str( metadata["Reason"]).ljust(100)) gLogger.notice(outStr)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.parseCommandLine() from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations allowUsers = Operations().getValue( "DataManagement/AllowUserReplicaManagement", False) from DIRAC.Core.Security.ProxyInfo import getProxyInfo res = getProxyInfo() if not res["OK"]: gLogger.fatal("Can't get proxy info", res["Message"]) dexit(1) properties = res["Value"].get("groupProperties", []) if not allowUsers: if "FileCatalogManagement" not in properties: gLogger.error( "You need to use a proxy from a group with FileCatalogManagement" ) dexit(5) from DIRAC.Resources.Catalog.FileCatalog import FileCatalog fc = FileCatalog() import os # parseCommandLine show help when mandatory arguments are not specified or incorrect argument args = Script.getPositionalArgs() inputFileName = args[0] if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() lfns = [lfn.strip() for lfn in string.splitlines()] inputFile.close() else: lfns = [inputFileName] res = fc.removeFile(lfns) if not res["OK"]: print("Error:", res["Message"]) dexit(1) for lfn in sorted(res["Value"]["Failed"].keys()): message = res["Value"]["Failed"][lfn] print("Error: failed to remove %s: %s" % (lfn, message)) print("Successfully removed %d catalog files." % (len(res["Value"]["Successful"])))
def main(): Script.registerSwitch("p:", "property=", "Add property to the user <name>=<value>") Script.registerSwitch("f", "force", "create the user if it doesn't exist") # Registering arguments will automatically add their description to the help menu Script.registerArgument(" user: User name") Script.registerArgument(" DN: DN of the User") Script.registerArgument(["group: Add the user to the group"]) Script.parseCommandLine(ignoreErrors=True) from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 forceCreation = False errorList = [] userProps = {} for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ("f", "force"): forceCreation = True elif unprocSw[0] in ("p", "property"): prop = unprocSw[1] pl = prop.split("=") if len(pl) < 2: errorList.append(( "in arguments", "Property %s has to include a '=' to separate name from value" % prop)) exitCode = 255 else: pName = pl[0] pValue = "=".join(pl[1:]) print("Setting property %s to %s" % (pName, pValue)) userProps[pName] = pValue userName, userProps["DN"], userProps["Groups"] = Script.getPositionalArgs( group=True) if not diracAdmin.csModifyUser( userName, userProps, createIfNonExistant=forceCreation): errorList.append(("modify user", "Cannot modify user %s" % userName)) exitCode = 255 else: result = diracAdmin.csCommitChanges() if not result["OK"]: errorList.append(("commit", result["Message"])) exitCode = 255 for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 3: error( "Missing all mandatory 'query', 'element', 'tableType' arguments") elif args[0].lower() not in ("select", "add", "modify", "delete"): error("Incorrect 'query' argument") elif args[1].lower() not in ("site", "resource", "component", "node"): error("Incorrect 'element' argument") elif args[2].lower() not in ("status", "log", "history"): error("Incorrect 'tableType' argument") else: query = args[0].lower() switches = dict(Script.getUnprocessedSwitches()) # Default values switches.setdefault("name", None) switches.setdefault("statusType", None) switches.setdefault("status", None) switches.setdefault("elementType", None) switches.setdefault("reason", None) switches.setdefault("lastCheckTime", None) switches.setdefault("tokenOwner", None) switches.setdefault("VO", None) if "status" in switches and switches["status"] is not None: switches["status"] = switches["status"].title() if switches["status"] not in ("Active", "Probing", "Degraded", "Banned", "Unknown", "Error"): error("'%s' is an invalid argument for switch 'status'" % switches["status"]) # when it's a add/modify query and status/reason/statusType are not specified # then some specific defaults are set up if query == "add" or query == "modify": if "status" not in switches or switches["status"] is None: switches["status"] = "Unknown" if "reason" not in switches or switches["reason"] is None: switches["reason"] = "Unknown reason" if "statusType" not in switches or switches["statusType"] is None: switches["statusType"] = "all" subLogger.debug("The switches used are:") map(subLogger.debug, switches.items()) return args, switches
def main(): Script.registerSwitch("e", "extended", "Show extended info") # Registering arguments will automatically add their description to the help menu Script.registerArgument(["Group: Only users from this group (default: all)"], default=["all"], mandatory=False) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs(group=True) import DIRAC from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() exitCode = 0 errorList = [] extendedInfo = False for unprocSw in Script.getUnprocessedSwitches(): if unprocSw[0] in ("e", "extended"): extendedInfo = True def printUsersInGroup(group=False): result = diracAdmin.csListUsers(group) if result["OK"]: if group: print("Users in group %s:" % group) else: print("All users registered:") for username in result["Value"]: print(" %s" % username) def describeUsersInGroup(group=False): result = diracAdmin.csListUsers(group) if result["OK"]: if group: print("Users in group %s:" % group) else: print("All users registered:") result = diracAdmin.csDescribeUsers(result["Value"]) print(diracAdmin.pPrint.pformat(result["Value"])) for group in args: if "all" in args: group = False if not extendedInfo: printUsersInGroup(group) else: describeUsersInGroup(group) for error in errorList: print("ERROR %s: %s" % error) DIRAC.exit(exitCode)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument( "LFN: Logical File Name or file containing LFNs") Script.registerArgument("Dest: Valid DIRAC SE") Script.registerArgument("Source: Valid DIRAC SE", default="", mandatory=False) Script.registerArgument("Cache: Local directory to be used as cache", default="", mandatory=False) _, args = Script.parseCommandLine(ignoreErrors=True) if len(args) > 4: Script.showHelp(exitCode=1) lfn, seName, sourceSE, localCache = Script.getPositionalArgs(group=True) from DIRAC.Interfaces.API.Dirac import Dirac dirac = Dirac() exitCode = 0 try: f = open(lfn, "r") lfns = f.read().splitlines() f.close() except Exception: lfns = [lfn] finalResult = {"Failed": [], "Successful": []} for lfn in lfns: result = dirac.replicateFile(lfn, seName, sourceSE, localCache, printOutput=True) if not result["OK"]: finalResult["Failed"].append(lfn) print("ERROR %s" % (result["Message"])) exitCode = 2 else: finalResult["Successful"].append(lfn) if len(lfns) > 1: print(finalResult) DIRAC.exit(exitCode)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.registerArgument(["Catalog: file catalog plug-ins"], mandatory=False) Script.parseCommandLine() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog import os # parseCommandLine show help when mandatory arguments are not specified or incorrect argument inputFileName, catalogs = Script.getPositionalArgs(group=True) if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() lfns = string.splitlines() inputFile.close() else: lfns = [inputFileName] res = FileCatalog(catalogs=catalogs).getFileMetadata(lfns) if not res["OK"]: print("ERROR:", res["Message"]) DIRACExit(-1) print("FileName".ljust(100), "Size".ljust(10), "GUID".ljust(40), "Status".ljust(8), "Checksum".ljust(10)) for lfn in sorted(res["Value"]["Successful"].keys()): metadata = res["Value"]["Successful"][lfn] checksum = "" if "Checksum" in metadata: checksum = str(metadata["Checksum"]) size = "" if "Size" in metadata: size = str(metadata["Size"]) guid = "" if "GUID" in metadata: guid = str(metadata["GUID"]) status = "" if "Status" in metadata: status = str(metadata["Status"]) print("%s %s %s %s %s" % (lfn.ljust(100), size.ljust(10), guid.ljust(40), status.ljust(8), checksum.ljust(10))) for lfn in sorted(res["Value"]["Failed"].keys()): message = res["Value"]["Failed"][lfn] print(lfn, message)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument("Setup: Name of the setup", default="", mandatory=False) Script.parseCommandLine(ignoreErrors=True) setup = Script.getPositionalArgs(group=True) from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin diracAdmin = DiracAdmin() result = diracAdmin.getServicePorts(setup, printOutput=True) if result["OK"]: DIRAC.exit(0) else: print(result["Message"]) DIRAC.exit(2)
def main(): global listOfFailedFiles Script.registerSwitch("D", "sync", "Make target directory identical to source") Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload") # Registering arguments will automatically add their description to the help menu Script.registerArgument( ( "LFN: Logical File Name (Path to directory)", "Path: Local path to the file (Path to directory)", ) ) Script.registerArgument( ( "Path: Local path to the file (Path to directory)", "LFN: Logical File Name (Path to directory)", ) ) Script.registerArgument(" SE: DIRAC Storage Element", mandatory=False) Script.parseCommandLine(ignoreErrors=False) args = Script.getPositionalArgs() if len(args) > 3: Script.showHelp() sync = False parallel = 1 for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "s" or switch[0].lower() == "sync": sync = True if switch[0].lower() == "j" or switch[0].lower() == "parallel": parallel = int(switch[1]) listOfFailedFiles = Manager().list() # This is the execution returnValue = run(args, sync, parallel) if listOfFailedFiles: gLogger.error("Some file operations failed:\n\t", "\n\t".join(listOfFailedFiles)) DIRAC.exit(1) if not returnValue["OK"]: gLogger.fatal(returnValue["Message"]) DIRAC.exit(1) else: gLogger.notice(returnValue["Value"]) DIRAC.exit(0)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.registerArgument(["LFN: Logical File Names"], mandatory=False) Script.parseCommandLine() import os import DIRAC from DIRAC import gLogger first, lfns = Script.getPositionalArgs(group=True) if os.path.exists(first): with open(first, "r") as inputFile: string = inputFile.read() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.insert(0, first) from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() errorReasons = {} successfullyRemoved = 0 for lfnList in breakListIntoChunks(lfns, 100): res = dm.removeFile(lfnList) if not res["OK"]: gLogger.error("Failed to remove data", res["Message"]) DIRAC.exit(-2) for lfn, r in res["Value"]["Failed"].items(): reason = str(r) if reason not in errorReasons: errorReasons[reason] = [] errorReasons[reason].append(lfn) successfullyRemoved += len(res["Value"]["Successful"]) for reason, lfns in errorReasons.items(): gLogger.notice("Failed to remove %d files with error: %s" % (len(lfns), reason)) if successfullyRemoved > 0: gLogger.notice("Successfully removed %d files" % successfullyRemoved) DIRAC.exit(0)
def main(): Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp() seName = args[1] fileName = args[0] import os from DIRAC import exit as DIRACExit, gLogger from DIRAC.Interfaces.API.Dirac import Dirac from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient stageLfns = {} if os.path.exists(fileName): try: lfnFile = open(fileName) lfns = [k.strip() for k in lfnFile.readlines()] lfnFile.close() except Exception: gLogger.exception("Can not open file", fileName) DIRACExit(-1) else: lfns = args[: len(args) - 1] stageLfns[seName] = lfns stagerClient = StorageManagerClient() res = stagerClient.setRequest( stageLfns, "WorkloadManagement", "updateJobFromStager@WorkloadManagement/JobStateUpdate", 0 ) # fake JobID = 0 if not res["OK"]: gLogger.error(res["Message"]) DIRACExit(-1) else: gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns) gLogger.notice("SE= %s" % seName) gLogger.notice("You can check their status and progress with dirac-stager-monitor-file <LFN> <SE>") DIRACExit()
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument("ShifterRole: Name of the shifter role, e.g. DataManager") Script.registerArgument("UserName: A user name, as registered in Registry section") Script.registerArgument("DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)") Script.parseCommandLine(ignoreErrors=True) csAPI = CSAPI() shifterRole, userName, diracGroup = Script.getPositionalArgs(group=True) res = csAPI.addShifter({shifterRole: {"User": userName, "Group": diracGroup}}) if not res["OK"]: gLogger.error("Could not add shifter", ": " + res["Message"]) DIRACExit(1) res = csAPI.commit() if not res["OK"]: gLogger.error("Could not add shifter", ": " + res["Message"]) DIRACExit(1) gLogger.notice("Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup))
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Name")) Script.registerArgument(" SE: Storage Element") Script.registerArgument(" status: status") Script.parseCommandLine() from DIRAC.Resources.Catalog.FileCatalog import FileCatalog catalog = FileCatalog() import os # parseCommandLine show help when mandatory arguments are not specified or incorrect argument inputFileName, se, newStatus = Script.getPositionalArgs(group=True) if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() lfns = string.splitlines() inputFile.close() else: lfns = [inputFileName] res = catalog.getReplicas(lfns, True) if not res["OK"]: print(res["Message"]) DIRACExit(-1) replicas = res["Value"]["Successful"] lfnDict = {} for lfn in lfns: lfnDict[lfn] = {} lfnDict[lfn]["SE"] = se lfnDict[lfn]["Status"] = newStatus lfnDict[lfn]["PFN"] = replicas[lfn][se] res = catalog.setReplicaStatus(lfnDict) if not res["OK"]: print("ERROR:", res["Message"]) if res["Value"]["Failed"]: print("Failed to update %d replica status" % len(res["Value"]["Failed"])) if res["Value"]["Successful"]: print("Successfully updated %d replica status" % len(res["Value"]["Successful"]))
def main(): # Instantiate the params class cliParams = Params() # Register accepted switches and their callbacks Script.registerSwitch("r", "showRaw", "show raw result from the query", cliParams.setRawResult) Script.registerSwitch("p:", "numPings=", "Number of pings to do (by default 1)", cliParams.setNumOfPingsToDo) Script.registerArgument(["System: system names"]) # Parse the command line and initialize DIRAC switches, servicesList = Script.parseCommandLine(ignoreErrors=False) # Get the list of services servicesList = Script.getPositionalArgs() # Do something! gLogger.notice("Ping %s!" % ", ".join(servicesList))
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) switches = dict(Script.getUnprocessedSwitches()) switches.setdefault("statusType", None) switches.setdefault("VO", None) for key in ("element", "name", "status", "reason"): if key not in switches: subLogger.error("%s Switch missing" % key) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) if not switches["element"] in ("Site", "Resource", "Node"): subLogger.error("Found %s as element switch" % switches["element"]) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) statuses = StateMachine.RSSMachine(None).getStates() if not switches["status"] in statuses: subLogger.error("Found %s as element switch" % switches["element"]) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") map(subLogger.debug, switches.items()) return switches
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument("TransID: transformation ID") Script.registerArgument(("LFN: LFN", "FileName: file containing LFNs")) Script.parseCommandLine() from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient tID, inputFileName = Script.getPositionalArgs(group=True) # get arguments lfns = [] if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() inputFile.close() lfns.extend([lfn.strip() for lfn in string.splitlines()]) else: lfns.append(inputFileName) tc = TransformationClient() res = tc.addFilesToTransformation(tID, lfns) # Files added here if not res["OK"]: DIRAC.gLogger.error(res["Message"]) DIRAC.exit(2) successfullyAdded = 0 alreadyPresent = 0 for lfn, message in res["Value"]["Successful"].items(): if message == "Added": successfullyAdded += 1 elif message == "Present": alreadyPresent += 1 if successfullyAdded > 0: DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded) if alreadyPresent > 0: DIRAC.gLogger.notice("Already present %d files" % alreadyPresent) DIRAC.exit(0)
def parseSwitches(): """ Parses the arguments passed by the user """ Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if args: subLogger.error( "Found the following positional args '%s', but we only accept switches" % args) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) switches = dict(Script.getUnprocessedSwitches()) for key in ("status", "se", "limit"): if key not in switches: subLogger.warn( "You're not using switch --%s, query may take long!" % key) if "status" in switches and switches["status"] not in ( "New", "Offline", "Waiting", "Failed", "StageSubmitted", "Staged", ): subLogger.error( 'Found "%s" as Status value. Incorrect value used!' % switches["status"]) subLogger.error("Please, check documentation below") Script.showHelp(exitCode=1) subLogger.debug("The switches used are:") map(subLogger.debug, switches.items()) return switches
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.registerArgument(["SE: Storage element"]) Script.parseCommandLine() from DIRAC.Core.Utilities.List import breakListIntoChunks from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() import os # parseCommandLine show help when mandatory arguments are not specified or incorrect argument first, storageElementNames = Script.getPositionalArgs(group=True) if os.path.exists(first): with open(first, "r") as inputFile: string = inputFile.read() lfns = [lfn.strip() for lfn in string.splitlines()] inputFile.close() else: lfns = [first] for lfnList in breakListIntoChunks(sorted(lfns, reverse=True), 500): for storageElementName in storageElementNames: res = dm.removeReplica(storageElementName, lfnList) if not res["OK"]: print("Error:", res["Message"]) continue for lfn in sorted(res["Value"]["Successful"]): print("Successfully removed %s replica of %s" % (storageElementName, lfn)) for lfn in sorted(res["Value"]["Failed"]): message = res["Value"]["Failed"][lfn] print("Error: failed to remove %s replica of %s: %s" % (storageElementName, lfn, message))
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Name")) Script.parseCommandLine() # parseCommandLine show help when mandatory arguments are not specified or incorrect argument inputFileName = Script.getPositionalArgs(group=True) if os.path.exists(inputFileName): lfns = [ lfn.strip().split()[0] for lfn in sorted(open(inputFileName, "r").read().splitlines()) ] else: lfns = [inputFileName] from DIRAC.DataManagementSystem.Client.DataManager import DataManager dm = DataManager() retVal = 0 for lfn in [lfn for lfn in lfns if lfn]: gLogger.notice("Cleaning directory %r ... " % lfn) result = dm.cleanLogicalDirectory(lfn) if not result["OK"]: gLogger.error("Failed to clean directory", result["Message"]) retVal = -1 else: if not result["Value"]["Failed"]: gLogger.notice("OK") else: for folder, message in result["Value"]["Failed"].items(): gLogger.error("Failed to clean folder", "%r: %s" % (folder, message)) retVal = -1 DIRACExit(retVal)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names")) Script.registerArgument(" SE: Storage element") Script.parseCommandLine() from DIRAC import gLogger from DIRAC.DataManagementSystem.Client.DataManager import DataManager # parseCommandLine show help when mandatory arguments are not specified or incorrect argument inputFileName, storageElement = Script.getPositionalArgs(group=True) if os.path.exists(inputFileName): inputFile = open(inputFileName, "r") string = inputFile.read() lfns = [lfn.strip() for lfn in string.splitlines()] inputFile.close() else: lfns = [inputFileName] res = DataManager().getReplicaMetadata(lfns, storageElement) if not res["OK"]: print("Error:", res["Message"]) DIRACExit(1) print("%s %s %s %s" % ("File".ljust(100), "Migrated".ljust(8), "Cached".ljust(8), "Size (bytes)".ljust(10))) for lfn, metadata in res["Value"]["Successful"].items(): print( "%s %s %s %s" % ( lfn.ljust(100), str(metadata["Migrated"]).ljust(8), str(metadata.get("Cached", metadata["Accessible"])).ljust(8), str(metadata["Size"]).ljust(10), ) ) for lfn, reason in res["Value"]["Failed"].items(): print("%s %s" % (lfn.ljust(100), reason.ljust(8)))