def parseSwitchesAndPositionalArguments():
"""
Parse switches and positional arguments given to the script
"""
# Parse the command line and initialize DIRAC
Script.parseCommandLine(ignoreErrors=False)
# Get arguments
allArgs = Script.getPositionalArgs()
gLogger.debug("All arguments: %s" % ", ".join(allArgs))
# Get unprocessed switches
switches = dict(Script.getUnprocessedSwitches())
gLogger.debug("The switches used are:")
map(gLogger.debug, switches.iteritems())
# Get grouped positional arguments
repType, user, services = Script.getPositionalArgs(group=True)
gLogger.debug("The positional arguments are:")
gLogger.debug("Report type:", repType)
gLogger.debug("Name or DN:", user)
gLogger.debug("Services:", services)
return switches, repType, user, services
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["LFN: Logical File Name or file containing LFNs"])
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
with open(lfns[0], "r") as f:
lfns = f.read().splitlines()
except Exception:
pass
result = dirac.getFile(lfns, printOutput=True)
if not result["OK"]:
print("ERROR %s" % (result["Message"]))
exitCode = 2
DIRAC.exit(exitCode)
def main():
Script.registerSwitch(
"v:",
"vo=",
"Location of pilot version in CS /Operations/<vo>/Pilot/Version"
" (default value specified in CS under /DIRAC/DefaultSetup)",
)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("version: pilot version you want to update to")
Script.parseCommandLine(ignoreErrors=False)
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
version = Script.getPositionalArgs(group=True)
vo = None
for switch in Script.getUnprocessedSwitches():
if switch[0] == "v" or switch[0] == "vo":
vo = switch[1]
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
def updatePilot(version, vo):
"""
Update in the CS the pilot version used,
If only one version present in CS it's overwritten.
If two versions present, the new one is added and the last removed
:param version: version vArBpC of pilot you want to use
:param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
"""
setup = vo
if not vo:
setup = gConfig.getValue("/DIRAC/DefaultSetup")
if not setup:
return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")
pilotVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup, [])
if not pilotVersion:
return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup)
pilotVersion.pop()
pilotVersion.insert(0, version)
api = CSAPI()
api.setOption("Operations/%s/Pilot/Version" % setup, ", ".join(pilotVersion))
result = api.commit()
if not result["OK"]:
gLogger.fatal("Could not commit new version of pilot!")
return result
newVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup)
return S_OK("New version of pilot set to %s" % newVersion)
result = updatePilot(version, vo)
if not result["OK"]:
gLogger.fatal(result["Message"])
DIRAC.exit(1)
gLogger.notice(result["Value"])
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
("LFN: LFN",
"File: File name containing a list of affected LFNs"))
Script.registerArgument(" SE: Name of Storage Element")
Script.registerArgument(" Status: New Status for the replica")
Script.parseCommandLine(ignoreErrors=False)
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName, storageElement, status = Script.getPositionalArgs(
group=True)
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
inputFile.close()
lfns = sorted(string.splitlines())
else:
lfns = [inputFileName]
fc = FileCatalog()
res = fc.getReplicas(lfns, allStatus=True)
if not res["OK"]:
gLogger.error("Failed to get catalog replicas.", res["Message"])
DIRAC.exit(-1)
lfnDict = {}
for lfn, error in res["Value"]["Failed"].items():
gLogger.error("Failed to get replicas for file.",
"%s:%s" % (lfn, error))
for lfn, replicas in res["Value"]["Successful"].items():
if storageElement not in replicas.keys():
gLogger.error("LFN not registered at provided storage element.",
"%s %s" % (lfn, storageElement))
else:
lfnDict[lfn] = {
"SE": storageElement,
"PFN": replicas[storageElement],
"Status": status
}
if not lfnDict:
gLogger.error("No files found at the supplied storage element.")
DIRAC.exit(2)
res = fc.setReplicaStatus(lfnDict)
if not res["OK"]:
gLogger.error("Failed to set catalog replica status.", res["Message"])
DIRAC.exit(-1)
for lfn, error in res["Value"]["Failed"].items():
gLogger.error("Failed to set replica status for file.",
"%s:%s" % (lfn, error))
gLogger.notice("Successfully updated the status of %d files at %s." %
(len(res["Value"]["Successful"].keys()), storageElement))
DIRAC.exit(0)
def main():
Script.registerArgument(["Agent: specify which agent to run"])
positionalArgs = Script.getPositionalArgs(group=True)
localCfg = Script.localCfg
agentName = positionalArgs[0]
localCfg.setConfigurationForAgent(agentName)
localCfg.addMandatoryEntry("/DIRAC/Setup")
localCfg.addDefaultEntry("/DIRAC/Security/UseServerCertificate", "yes")
localCfg.addDefaultEntry("LogLevel", "INFO")
localCfg.addDefaultEntry("LogColor", True)
resultDict = localCfg.loadUserData()
if not resultDict["OK"]:
gLogger.error("There were errors when loading configuration",
resultDict["Message"])
sys.exit(1)
includeExtensionErrors()
agentReactor = AgentReactor(positionalArgs[0])
result = agentReactor.loadAgentModules(positionalArgs)
if result["OK"]:
agentReactor.go()
# dirac_agent might interact with ARC library which cannot be closed using a simple sys.exit(0)
# See https://bugzilla.nordugrid.org/show_bug.cgi?id=4022 for further details
os._exit(0)
else:
gLogger.error("Error while loading agent module", result["Message"])
sys.exit(2)
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ":" if short else "",
longOption + "=", doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose("Found switch %r with value %r" %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches["DryRun"] = not self.switches.get("Execute", False)
self.switches["SourceSE"] = self.switches.get("SourceSE",
"").split(",")
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("element", None)
switches.setdefault("defaultStatus", "Banned")
if not switches["element"] in ("all", "Site", "Resource", "Node", None):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ":" if short else "",
longOption + "=", doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
ops = Operations()
if not ops.getValue("DataManagement/ArchiveFiles/Enabled", False):
sLog.error(
'The "ArchiveFiles" operation is not enabled, contact your administrator!'
)
DIRAC.exit(1)
for _short, longOption, _doc in self.options:
defaultValue = ops.getValue(
"DataManagement/ArchiveFiles/%s" % longOption, None)
if defaultValue:
sLog.verbose(
"Found default value in the CS for %r with value %r" %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for _short, longOption, _doc in self.flags:
defaultValue = ops.getValue(
"DataManagement/ArchiveFiles/%s" % longOption, False)
if defaultValue:
sLog.verbose(
"Found default value in the CS for %r with value %r" %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose("Found switch %r with value %r" %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches["DryRun"] = not self.switches.get("Execute", False)
self.switches["SourceSE"] = self.switches.get("SourceSE",
"").split(",")
def main():
global overwrite
Script.registerSwitch("f", "force", "Force overwrite of existing file",
setOverwrite)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 4:
Script.showHelp(exitCode=1)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
lfns = []
if len(args) == 1:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
for line in inputFile:
line = line.rstrip()
items = line.split()
items[0] = items[0].replace("LFN:", "").replace("lfn:", "")
lfns.append(getDict(items))
inputFile.close()
else:
gLogger.error("Error: LFN list '%s' missing." % inputFileName)
exitCode = 4
else:
lfns.append(getDict(args))
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn["localfile"]):
gLogger.error("File %s must exist locally" % lfn["localfile"])
exitCode = 1
continue
if not os.path.isfile(lfn["localfile"]):
gLogger.error("%s is not a file" % lfn["localfile"])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn["lfn"])
res = dm.putAndRegister(lfn["lfn"],
lfn["localfile"],
lfn["SE"],
lfn["guid"],
overwrite=overwrite)
if not res["OK"]:
exitCode = 3
gLogger.error("Error: failed to upload %s to %s: %s" %
(lfn["lfn"], lfn["SE"], res))
continue
else:
gLogger.notice("Successfully uploaded file to %s" % lfn["SE"])
DIRAC.exit(exitCode)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("prodID: Production ID")
Script.registerArgument("transID: Transformation ID")
Script.registerArgument("parentTransID: Parent Transformation ID", default="", mandatory=False)
_, args = Script.parseCommandLine()
from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
prodClient = ProductionClient()
transClient = TransformationClient()
# get arguments
prodID, transID, parentTransID = Script.getPositionalArgs(group=True)
if len(args) > 3:
Script.showHelp(exitCode=1)
res = transClient.getTransformation(transID)
if not res["OK"]:
DIRAC.gLogger.error("Failed to get transformation %s: %s" % (transID, res["Message"]))
DIRAC.exit(-1)
transID = res["Value"]["TransformationID"]
if parentTransID:
res = transClient.getTransformation(parentTransID)
if not res["OK"]:
DIRAC.gLogger.error("Failed to get transformation %s: %s" % (parentTransID, res["Message"]))
DIRAC.exit(-1)
parentTransID = res["Value"]["TransformationID"]
res = prodClient.getProduction(prodID)
if not res["OK"]:
DIRAC.gLogger.error("Failed to get production %s: %s" % (prodID, res["Message"]))
DIRAC.exit(-1)
prodID = res["Value"]["ProductionID"]
res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID)
if not res["OK"]:
DIRAC.gLogger.error(res["Message"])
DIRAC.exit(-1)
if parentTransID:
msg = "Transformation %s successfully added to production %s with parent transformation %s" % (
transID,
prodID,
parentTransID,
)
else:
msg = "Transformation %s successfully added to production %s with no parent transformation" % (transID, prodID)
DIRAC.gLogger.notice(msg)
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"Request: ID of the Stage request in the StorageManager")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if not len(args) == 1:
Script.showHelp()
from DIRAC import exit as DIRACExit, gLogger
try:
taskID = int(args[0])
except Exception:
gLogger.fatal("Stage requestID must be an integer")
DIRACExit(2)
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
client = StorageManagerClient()
res = client.getTaskSummary(taskID)
if not res["OK"]:
gLogger.error(res["Message"])
DIRACExit(2)
taskInfo = res["Value"]["TaskInfo"]
replicaInfo = res["Value"]["ReplicaInfo"]
outStr = "%s: %s" % ("TaskID".ljust(20), taskID)
outStr += "\n%s: %s" % ("Status".ljust(20), taskInfo[taskID]["Status"])
outStr += "\n%s: %s" % ("Source".ljust(20), taskInfo[taskID]["Source"])
outStr += "\n%s: %s" % ("SourceTaskID".ljust(20),
taskInfo[taskID]["SourceTaskID"])
outStr += "\n%s: %s" % ("CallBackMethod".ljust(20),
taskInfo[taskID]["CallBackMethod"])
outStr += "\n%s: %s" % ("SubmitTime".ljust(20),
taskInfo[taskID]["SubmitTime"])
outStr += "\n%s: %s" % ("CompleteTime".ljust(20),
taskInfo[taskID]["CompleteTime"])
for lfn, metadata in replicaInfo.items():
outStr += "\n"
outStr += "\n\t%s: %s" % ("LFN".ljust(8), lfn.ljust(100))
outStr += "\n\t%s: %s" % ("SE".ljust(8),
metadata["StorageElement"].ljust(100))
outStr += "\n\t%s: %s" % ("PFN".ljust(8), str(
metadata["PFN"]).ljust(100))
outStr += "\n\t%s: %s" % ("Size".ljust(8), str(
metadata["FileSize"]).ljust(100))
outStr += "\n\t%s: %s" % ("Status".ljust(8),
metadata["Status"].ljust(100))
outStr += "\n\t%s: %s" % ("Reason".ljust(8), str(
metadata["Reason"]).ljust(100))
gLogger.notice(outStr)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.parseCommandLine()
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
allowUsers = Operations().getValue(
"DataManagement/AllowUserReplicaManagement", False)
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
res = getProxyInfo()
if not res["OK"]:
gLogger.fatal("Can't get proxy info", res["Message"])
dexit(1)
properties = res["Value"].get("groupProperties", [])
if not allowUsers:
if "FileCatalogManagement" not in properties:
gLogger.error(
"You need to use a proxy from a group with FileCatalogManagement"
)
dexit(5)
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
fc = FileCatalog()
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = fc.removeFile(lfns)
if not res["OK"]:
print("Error:", res["Message"])
dexit(1)
for lfn in sorted(res["Value"]["Failed"].keys()):
message = res["Value"]["Failed"][lfn]
print("Error: failed to remove %s: %s" % (lfn, message))
print("Successfully removed %d catalog files." %
(len(res["Value"]["Successful"])))
def main():
Script.registerSwitch("p:", "property=",
"Add property to the user <name>=<value>")
Script.registerSwitch("f", "force", "create the user if it doesn't exist")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" user: User name")
Script.registerArgument(" DN: DN of the User")
Script.registerArgument(["group: Add the user to the group"])
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
forceCreation = False
errorList = []
userProps = {}
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("f", "force"):
forceCreation = True
elif unprocSw[0] in ("p", "property"):
prop = unprocSw[1]
pl = prop.split("=")
if len(pl) < 2:
errorList.append((
"in arguments",
"Property %s has to include a '=' to separate name from value"
% prop))
exitCode = 255
else:
pName = pl[0]
pValue = "=".join(pl[1:])
print("Setting property %s to %s" % (pName, pValue))
userProps[pName] = pValue
userName, userProps["DN"], userProps["Groups"] = Script.getPositionalArgs(
group=True)
if not diracAdmin.csModifyUser(
userName, userProps, createIfNonExistant=forceCreation):
errorList.append(("modify user", "Cannot modify user %s" % userName))
exitCode = 255
else:
result = diracAdmin.csCommitChanges()
if not result["OK"]:
errorList.append(("commit", result["Message"]))
exitCode = 255
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
error(
"Missing all mandatory 'query', 'element', 'tableType' arguments")
elif args[0].lower() not in ("select", "add", "modify", "delete"):
error("Incorrect 'query' argument")
elif args[1].lower() not in ("site", "resource", "component", "node"):
error("Incorrect 'element' argument")
elif args[2].lower() not in ("status", "log", "history"):
error("Incorrect 'tableType' argument")
else:
query = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("name", None)
switches.setdefault("statusType", None)
switches.setdefault("status", None)
switches.setdefault("elementType", None)
switches.setdefault("reason", None)
switches.setdefault("lastCheckTime", None)
switches.setdefault("tokenOwner", None)
switches.setdefault("VO", None)
if "status" in switches and switches["status"] is not None:
switches["status"] = switches["status"].title()
if switches["status"] not in ("Active", "Probing", "Degraded",
"Banned", "Unknown", "Error"):
error("'%s' is an invalid argument for switch 'status'" %
switches["status"])
# when it's a add/modify query and status/reason/statusType are not specified
# then some specific defaults are set up
if query == "add" or query == "modify":
if "status" not in switches or switches["status"] is None:
switches["status"] = "Unknown"
if "reason" not in switches or switches["reason"] is None:
switches["reason"] = "Unknown reason"
if "statusType" not in switches or switches["statusType"] is None:
switches["statusType"] = "all"
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return args, switches
def main():
Script.registerSwitch("e", "extended", "Show extended info")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["Group: Only users from this group (default: all)"], default=["all"], mandatory=False)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs(group=True)
import DIRAC
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
extendedInfo = False
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("e", "extended"):
extendedInfo = True
def printUsersInGroup(group=False):
result = diracAdmin.csListUsers(group)
if result["OK"]:
if group:
print("Users in group %s:" % group)
else:
print("All users registered:")
for username in result["Value"]:
print(" %s" % username)
def describeUsersInGroup(group=False):
result = diracAdmin.csListUsers(group)
if result["OK"]:
if group:
print("Users in group %s:" % group)
else:
print("All users registered:")
result = diracAdmin.csDescribeUsers(result["Value"])
print(diracAdmin.pPrint.pformat(result["Value"]))
for group in args:
if "all" in args:
group = False
if not extendedInfo:
printUsersInGroup(group)
else:
describeUsersInGroup(group)
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"LFN: Logical File Name or file containing LFNs")
Script.registerArgument("Dest: Valid DIRAC SE")
Script.registerArgument("Source: Valid DIRAC SE",
default="",
mandatory=False)
Script.registerArgument("Cache: Local directory to be used as cache",
default="",
mandatory=False)
_, args = Script.parseCommandLine(ignoreErrors=True)
if len(args) > 4:
Script.showHelp(exitCode=1)
lfn, seName, sourceSE, localCache = Script.getPositionalArgs(group=True)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
try:
f = open(lfn, "r")
lfns = f.read().splitlines()
f.close()
except Exception:
lfns = [lfn]
finalResult = {"Failed": [], "Successful": []}
for lfn in lfns:
result = dirac.replicateFile(lfn,
seName,
sourceSE,
localCache,
printOutput=True)
if not result["OK"]:
finalResult["Failed"].append(lfn)
print("ERROR %s" % (result["Message"]))
exitCode = 2
else:
finalResult["Successful"].append(lfn)
if len(lfns) > 1:
print(finalResult)
DIRAC.exit(exitCode)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names"))
Script.registerArgument(["Catalog: file catalog plug-ins"], mandatory=False)
Script.parseCommandLine()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName, catalogs = Script.getPositionalArgs(group=True)
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = string.splitlines()
inputFile.close()
else:
lfns = [inputFileName]
res = FileCatalog(catalogs=catalogs).getFileMetadata(lfns)
if not res["OK"]:
print("ERROR:", res["Message"])
DIRACExit(-1)
print("FileName".ljust(100), "Size".ljust(10), "GUID".ljust(40), "Status".ljust(8), "Checksum".ljust(10))
for lfn in sorted(res["Value"]["Successful"].keys()):
metadata = res["Value"]["Successful"][lfn]
checksum = ""
if "Checksum" in metadata:
checksum = str(metadata["Checksum"])
size = ""
if "Size" in metadata:
size = str(metadata["Size"])
guid = ""
if "GUID" in metadata:
guid = str(metadata["GUID"])
status = ""
if "Status" in metadata:
status = str(metadata["Status"])
print("%s %s %s %s %s" % (lfn.ljust(100), size.ljust(10), guid.ljust(40), status.ljust(8), checksum.ljust(10)))
for lfn in sorted(res["Value"]["Failed"].keys()):
message = res["Value"]["Failed"][lfn]
print(lfn, message)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("Setup: Name of the setup",
default="",
mandatory=False)
Script.parseCommandLine(ignoreErrors=True)
setup = Script.getPositionalArgs(group=True)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
result = diracAdmin.getServicePorts(setup, printOutput=True)
if result["OK"]:
DIRAC.exit(0)
else:
print(result["Message"])
DIRAC.exit(2)
def main():
global listOfFailedFiles
Script.registerSwitch("D", "sync", "Make target directory identical to source")
Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
(
"LFN: Logical File Name (Path to directory)",
"Path: Local path to the file (Path to directory)",
)
)
Script.registerArgument(
(
"Path: Local path to the file (Path to directory)",
"LFN: Logical File Name (Path to directory)",
)
)
Script.registerArgument(" SE: DIRAC Storage Element", mandatory=False)
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) > 3:
Script.showHelp()
sync = False
parallel = 1
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "s" or switch[0].lower() == "sync":
sync = True
if switch[0].lower() == "j" or switch[0].lower() == "parallel":
parallel = int(switch[1])
listOfFailedFiles = Manager().list()
# This is the execution
returnValue = run(args, sync, parallel)
if listOfFailedFiles:
gLogger.error("Some file operations failed:\n\t", "\n\t".join(listOfFailedFiles))
DIRAC.exit(1)
if not returnValue["OK"]:
gLogger.fatal(returnValue["Message"])
DIRAC.exit(1)
else:
gLogger.notice(returnValue["Value"])
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.registerArgument(["LFN: Logical File Names"], mandatory=False)
Script.parseCommandLine()
import os
import DIRAC
from DIRAC import gLogger
first, lfns = Script.getPositionalArgs(group=True)
if os.path.exists(first):
with open(first, "r") as inputFile:
string = inputFile.read()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.insert(0, first)
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
errorReasons = {}
successfullyRemoved = 0
for lfnList in breakListIntoChunks(lfns, 100):
res = dm.removeFile(lfnList)
if not res["OK"]:
gLogger.error("Failed to remove data", res["Message"])
DIRAC.exit(-2)
for lfn, r in res["Value"]["Failed"].items():
reason = str(r)
if reason not in errorReasons:
errorReasons[reason] = []
errorReasons[reason].append(lfn)
successfullyRemoved += len(res["Value"]["Successful"])
for reason, lfns in errorReasons.items():
gLogger.notice("Failed to remove %d files with error: %s" %
(len(lfns), reason))
if successfullyRemoved > 0:
gLogger.notice("Successfully removed %d files" % successfullyRemoved)
DIRAC.exit(0)
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
seName = args[1]
fileName = args[0]
import os
from DIRAC import exit as DIRACExit, gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
stageLfns = {}
if os.path.exists(fileName):
try:
lfnFile = open(fileName)
lfns = [k.strip() for k in lfnFile.readlines()]
lfnFile.close()
except Exception:
gLogger.exception("Can not open file", fileName)
DIRACExit(-1)
else:
lfns = args[: len(args) - 1]
stageLfns[seName] = lfns
stagerClient = StorageManagerClient()
res = stagerClient.setRequest(
stageLfns, "WorkloadManagement", "updateJobFromStager@WorkloadManagement/JobStateUpdate", 0
) # fake JobID = 0
if not res["OK"]:
gLogger.error(res["Message"])
DIRACExit(-1)
else:
gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns)
gLogger.notice("SE= %s" % seName)
gLogger.notice("You can check their status and progress with dirac-stager-monitor-file <LFN> <SE>")
DIRACExit()
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("ShifterRole: Name of the shifter role, e.g. DataManager")
Script.registerArgument("UserName: A user name, as registered in Registry section")
Script.registerArgument("DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)")
Script.parseCommandLine(ignoreErrors=True)
csAPI = CSAPI()
shifterRole, userName, diracGroup = Script.getPositionalArgs(group=True)
res = csAPI.addShifter({shifterRole: {"User": userName, "Group": diracGroup}})
if not res["OK"]:
gLogger.error("Could not add shifter", ": " + res["Message"])
DIRACExit(1)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Could not add shifter", ": " + res["Message"])
DIRACExit(1)
gLogger.notice("Added shifter %s as user %s with group %s" % (shifterRole, userName, diracGroup))
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Name"))
Script.registerArgument(" SE: Storage Element")
Script.registerArgument(" status: status")
Script.parseCommandLine()
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
catalog = FileCatalog()
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName, se, newStatus = Script.getPositionalArgs(group=True)
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = string.splitlines()
inputFile.close()
else:
lfns = [inputFileName]
res = catalog.getReplicas(lfns, True)
if not res["OK"]:
print(res["Message"])
DIRACExit(-1)
replicas = res["Value"]["Successful"]
lfnDict = {}
for lfn in lfns:
lfnDict[lfn] = {}
lfnDict[lfn]["SE"] = se
lfnDict[lfn]["Status"] = newStatus
lfnDict[lfn]["PFN"] = replicas[lfn][se]
res = catalog.setReplicaStatus(lfnDict)
if not res["OK"]:
print("ERROR:", res["Message"])
if res["Value"]["Failed"]:
print("Failed to update %d replica status" % len(res["Value"]["Failed"]))
if res["Value"]["Successful"]:
print("Successfully updated %d replica status" % len(res["Value"]["Successful"]))
def main():
# Instantiate the params class
cliParams = Params()
# Register accepted switches and their callbacks
Script.registerSwitch("r", "showRaw", "show raw result from the query",
cliParams.setRawResult)
Script.registerSwitch("p:", "numPings=",
"Number of pings to do (by default 1)",
cliParams.setNumOfPingsToDo)
Script.registerArgument(["System: system names"])
# Parse the command line and initialize DIRAC
switches, servicesList = Script.parseCommandLine(ignoreErrors=False)
# Get the list of services
servicesList = Script.getPositionalArgs()
# Do something!
gLogger.notice("Ping %s!" % ", ".join(servicesList))
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
switches.setdefault("statusType", None)
switches.setdefault("VO", None)
for key in ("element", "name", "status", "reason"):
if key not in switches:
subLogger.error("%s Switch missing" % key)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
if not switches["element"] in ("Site", "Resource", "Node"):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
statuses = StateMachine.RSSMachine(None).getStates()
if not switches["status"] in statuses:
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("TransID: transformation ID")
Script.registerArgument(("LFN: LFN", "FileName: file containing LFNs"))
Script.parseCommandLine()
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tID, inputFileName = Script.getPositionalArgs(group=True)
# get arguments
lfns = []
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
inputFile.close()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
tc = TransformationClient()
res = tc.addFilesToTransformation(tID, lfns) # Files added here
if not res["OK"]:
DIRAC.gLogger.error(res["Message"])
DIRAC.exit(2)
successfullyAdded = 0
alreadyPresent = 0
for lfn, message in res["Value"]["Successful"].items():
if message == "Added":
successfullyAdded += 1
elif message == "Present":
alreadyPresent += 1
if successfullyAdded > 0:
DIRAC.gLogger.notice("Successfully added %d files" % successfullyAdded)
if alreadyPresent > 0:
DIRAC.gLogger.notice("Already present %d files" % alreadyPresent)
DIRAC.exit(0)
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
for key in ("status", "se", "limit"):
if key not in switches:
subLogger.warn(
"You're not using switch --%s, query may take long!" % key)
if "status" in switches and switches["status"] not in (
"New",
"Offline",
"Waiting",
"Failed",
"StageSubmitted",
"Staged",
):
subLogger.error(
'Found "%s" as Status value. Incorrect value used!' %
switches["status"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Names"))
Script.registerArgument(["SE: Storage element"])
Script.parseCommandLine()
from DIRAC.Core.Utilities.List import breakListIntoChunks
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
import os
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
first, storageElementNames = Script.getPositionalArgs(group=True)
if os.path.exists(first):
with open(first, "r") as inputFile:
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [first]
for lfnList in breakListIntoChunks(sorted(lfns, reverse=True), 500):
for storageElementName in storageElementNames:
res = dm.removeReplica(storageElementName, lfnList)
if not res["OK"]:
print("Error:", res["Message"])
continue
for lfn in sorted(res["Value"]["Successful"]):
print("Successfully removed %s replica of %s" %
(storageElementName, lfn))
for lfn in sorted(res["Value"]["Failed"]):
message = res["Value"]["Failed"][lfn]
print("Error: failed to remove %s replica of %s: %s" %
(storageElementName, lfn, message))
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs",
"LFN: Logical File Name"))
Script.parseCommandLine()
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName = Script.getPositionalArgs(group=True)
if os.path.exists(inputFileName):
lfns = [
lfn.strip().split()[0]
for lfn in sorted(open(inputFileName, "r").read().splitlines())
]
else:
lfns = [inputFileName]
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
dm = DataManager()
retVal = 0
for lfn in [lfn for lfn in lfns if lfn]:
gLogger.notice("Cleaning directory %r ... " % lfn)
result = dm.cleanLogicalDirectory(lfn)
if not result["OK"]:
gLogger.error("Failed to clean directory", result["Message"])
retVal = -1
else:
if not result["Value"]["Failed"]:
gLogger.notice("OK")
else:
for folder, message in result["Value"]["Failed"].items():
gLogger.error("Failed to clean folder",
"%r: %s" % (folder, message))
retVal = -1
DIRACExit(retVal)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(("LocalFile: Path to local file containing LFNs", "LFN: Logical File Names"))
Script.registerArgument(" SE: Storage element")
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
inputFileName, storageElement = Script.getPositionalArgs(group=True)
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
string = inputFile.read()
lfns = [lfn.strip() for lfn in string.splitlines()]
inputFile.close()
else:
lfns = [inputFileName]
res = DataManager().getReplicaMetadata(lfns, storageElement)
if not res["OK"]:
print("Error:", res["Message"])
DIRACExit(1)
print("%s %s %s %s" % ("File".ljust(100), "Migrated".ljust(8), "Cached".ljust(8), "Size (bytes)".ljust(10)))
for lfn, metadata in res["Value"]["Successful"].items():
print(
"%s %s %s %s"
% (
lfn.ljust(100),
str(metadata["Migrated"]).ljust(8),
str(metadata.get("Cached", metadata["Accessible"])).ljust(8),
str(metadata["Size"]).ljust(10),
)
)
for lfn, reason in res["Value"]["Failed"].items():
print("%s %s" % (lfn.ljust(100), reason.ljust(8)))