def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["LFN: Logical File Name or file containing LFNs"])
Script.parseCommandLine(ignoreErrors=True)
lfns = Script.getPositionalArgs()
if len(lfns) < 1:
Script.showHelp()
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
if len(lfns) == 1:
try:
with open(lfns[0], "r") as f:
lfns = f.read().splitlines()
except Exception:
pass
result = dirac.getFile(lfns, printOutput=True)
if not result["OK"]:
print("ERROR %s" % (result["Message"]))
exitCode = 2
DIRAC.exit(exitCode)
def getBoolean(value):
if value.lower() == "true":
return True
elif value.lower() == "false":
return False
else:
Script.showHelp()
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ":" if short else "",
longOption + "=", doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose("Found switch %r with value %r" %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches["DryRun"] = not self.switches.get("Execute", False)
self.switches["SourceSE"] = self.switches.get("SourceSE",
"").split(",")
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("element", None)
switches.setdefault("defaultStatus", "Banned")
if not switches["element"] in ("all", "Site", "Resource", "Node", None):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def main():
Script.registerSwitch(
"", "Site=", "Site for which protocols are to be set (mandatory)")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["Protocol: SE access protocol"], mandatory=False)
switches, args = Script.parseCommandLine(ignoreErrors=True)
site = None
for switch in switches:
if switch[0].lower() == "site":
site = switch[1]
if not site or not args:
Script.showHelp(exitCode=1)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
result = diracAdmin.setSiteProtocols(site, args, printOutput=True)
if not result["OK"]:
print("ERROR: %s" % result["Message"])
exitCode = 2
DIRAC.exit(exitCode)
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ":" if short else "",
longOption + "=", doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
ops = Operations()
if not ops.getValue("DataManagement/ArchiveFiles/Enabled", False):
sLog.error(
'The "ArchiveFiles" operation is not enabled, contact your administrator!'
)
DIRAC.exit(1)
for _short, longOption, _doc in self.options:
defaultValue = ops.getValue(
"DataManagement/ArchiveFiles/%s" % longOption, None)
if defaultValue:
sLog.verbose(
"Found default value in the CS for %r with value %r" %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for _short, longOption, _doc in self.flags:
defaultValue = ops.getValue(
"DataManagement/ArchiveFiles/%s" % longOption, False)
if defaultValue:
sLog.verbose(
"Found default value in the CS for %r with value %r" %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose("Found switch %r with value %r" %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches["DryRun"] = not self.switches.get("Execute", False)
self.switches["SourceSE"] = self.switches.get("SourceSE",
"").split(",")
def main():
global overwrite
Script.registerSwitch("f", "force", "Force overwrite of existing file",
setOverwrite)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 1 or len(args) > 4:
Script.showHelp(exitCode=1)
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
from DIRAC import gLogger
import DIRAC
exitCode = 0
lfns = []
if len(args) == 1:
inputFileName = args[0]
if os.path.exists(inputFileName):
inputFile = open(inputFileName, "r")
for line in inputFile:
line = line.rstrip()
items = line.split()
items[0] = items[0].replace("LFN:", "").replace("lfn:", "")
lfns.append(getDict(items))
inputFile.close()
else:
gLogger.error("Error: LFN list '%s' missing." % inputFileName)
exitCode = 4
else:
lfns.append(getDict(args))
dm = DataManager()
for lfn in lfns:
if not os.path.exists(lfn["localfile"]):
gLogger.error("File %s must exist locally" % lfn["localfile"])
exitCode = 1
continue
if not os.path.isfile(lfn["localfile"]):
gLogger.error("%s is not a file" % lfn["localfile"])
exitCode = 2
continue
gLogger.notice("\nUploading %s" % lfn["lfn"])
res = dm.putAndRegister(lfn["lfn"],
lfn["localfile"],
lfn["SE"],
lfn["guid"],
overwrite=overwrite)
if not res["OK"]:
exitCode = 3
gLogger.error("Error: failed to upload %s to %s: %s" %
(lfn["lfn"], lfn["SE"], res))
continue
else:
gLogger.notice("Successfully uploaded file to %s" % lfn["SE"])
DIRAC.exit(exitCode)
def main():
global groupName
global groupProperties
global userNames
Script.registerSwitch("G:", "GroupName:", "Name of the Group (Mandatory)", setGroupName)
Script.registerSwitch(
"U:", "UserName:"******"Short Name of user to be added to the Group (Allow Multiple instances or None)", addUserName
)
Script.registerSwitch(
"P:", "Property:", "Property to be added to the Group (Allow Multiple instances or None)", addProperty
)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
["Property=<Value>: Other properties to be added to the Group like (VOMSRole=XXXX)"], mandatory=False
)
_, args = Script.parseCommandLine(ignoreErrors=True)
if groupName is None:
Script.showHelp(exitCode=1)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
groupProps = {}
if userNames:
groupProps["Users"] = ", ".join(userNames)
if groupProperties:
groupProps["Properties"] = ", ".join(groupProperties)
for prop in args:
pl = prop.split("=")
if len(pl) < 2:
errorList.append(("in arguments", "Property %s has to include a '=' to separate name from value" % prop))
exitCode = 255
else:
pName = pl[0]
pValue = "=".join(pl[1:])
gLogger.info("Setting property %s to %s" % (pName, pValue))
groupProps[pName] = pValue
if not diracAdmin.csModifyGroup(groupName, groupProps, createIfNonExistant=True)["OK"]:
errorList.append(("add group", "Cannot register group %s" % groupName))
exitCode = 255
else:
result = diracAdmin.csCommitChanges()
if not result["OK"]:
errorList.append(("commit", result["Message"]))
exitCode = 255
for error in errorList:
gLogger.error("%s: %s" % error)
DIRAC.exit(exitCode)
def error(msg):
"""
Format error messages
"""
subLogger.error("\nERROR:")
subLogger.error("\t" + msg)
subLogger.error("\tPlease, check documentation below")
Script.showHelp(exitCode=1)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("prodID: Production ID")
Script.registerArgument("transID: Transformation ID")
Script.registerArgument("parentTransID: Parent Transformation ID", default="", mandatory=False)
_, args = Script.parseCommandLine()
from DIRAC.ProductionSystem.Client.ProductionClient import ProductionClient
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
prodClient = ProductionClient()
transClient = TransformationClient()
# get arguments
prodID, transID, parentTransID = Script.getPositionalArgs(group=True)
if len(args) > 3:
Script.showHelp(exitCode=1)
res = transClient.getTransformation(transID)
if not res["OK"]:
DIRAC.gLogger.error("Failed to get transformation %s: %s" % (transID, res["Message"]))
DIRAC.exit(-1)
transID = res["Value"]["TransformationID"]
if parentTransID:
res = transClient.getTransformation(parentTransID)
if not res["OK"]:
DIRAC.gLogger.error("Failed to get transformation %s: %s" % (parentTransID, res["Message"]))
DIRAC.exit(-1)
parentTransID = res["Value"]["TransformationID"]
res = prodClient.getProduction(prodID)
if not res["OK"]:
DIRAC.gLogger.error("Failed to get production %s: %s" % (prodID, res["Message"]))
DIRAC.exit(-1)
prodID = res["Value"]["ProductionID"]
res = prodClient.addTransformationsToProduction(prodID, transID, parentTransID)
if not res["OK"]:
DIRAC.gLogger.error(res["Message"])
DIRAC.exit(-1)
if parentTransID:
msg = "Transformation %s successfully added to production %s with parent transformation %s" % (
transID,
prodID,
parentTransID,
)
else:
msg = "Transformation %s successfully added to production %s with no parent transformation" % (transID, prodID)
DIRAC.gLogger.notice(msg)
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"Request: ID of the Stage request in the StorageManager")
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if not len(args) == 1:
Script.showHelp()
from DIRAC import exit as DIRACExit, gLogger
try:
taskID = int(args[0])
except Exception:
gLogger.fatal("Stage requestID must be an integer")
DIRACExit(2)
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
client = StorageManagerClient()
res = client.getTaskSummary(taskID)
if not res["OK"]:
gLogger.error(res["Message"])
DIRACExit(2)
taskInfo = res["Value"]["TaskInfo"]
replicaInfo = res["Value"]["ReplicaInfo"]
outStr = "%s: %s" % ("TaskID".ljust(20), taskID)
outStr += "\n%s: %s" % ("Status".ljust(20), taskInfo[taskID]["Status"])
outStr += "\n%s: %s" % ("Source".ljust(20), taskInfo[taskID]["Source"])
outStr += "\n%s: %s" % ("SourceTaskID".ljust(20),
taskInfo[taskID]["SourceTaskID"])
outStr += "\n%s: %s" % ("CallBackMethod".ljust(20),
taskInfo[taskID]["CallBackMethod"])
outStr += "\n%s: %s" % ("SubmitTime".ljust(20),
taskInfo[taskID]["SubmitTime"])
outStr += "\n%s: %s" % ("CompleteTime".ljust(20),
taskInfo[taskID]["CompleteTime"])
for lfn, metadata in replicaInfo.items():
outStr += "\n"
outStr += "\n\t%s: %s" % ("LFN".ljust(8), lfn.ljust(100))
outStr += "\n\t%s: %s" % ("SE".ljust(8),
metadata["StorageElement"].ljust(100))
outStr += "\n\t%s: %s" % ("PFN".ljust(8), str(
metadata["PFN"]).ljust(100))
outStr += "\n\t%s: %s" % ("Size".ljust(8), str(
metadata["FileSize"]).ljust(100))
outStr += "\n\t%s: %s" % ("Status".ljust(8),
metadata["Status"].ljust(100))
outStr += "\n\t%s: %s" % ("Reason".ljust(8), str(
metadata["Reason"]).ljust(100))
gLogger.notice(outStr)
def main():
Script.registerSwitch("f:", "File=",
"Get status for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=",
"Get status for jobs in the given group")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["JobID: DIRAC Job ID"], mandatory=False)
sws, args = Script.parseCommandLine(ignoreErrors=True)
from DIRAC import exit as DIRACExit
from DIRAC.Core.Utilities.Time import toString, date, day
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
dirac = Dirac()
exitCode = 0
jobs = []
for key, value in sws:
if key.lower() in ("f", "file"):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif key.lower() in ("g", "jobgroup"):
jobDate = toString(date() - 30 * day)
# Choose jobs no more than 30 days old
result = dirac.selectJobs(jobGroup=value, date=jobDate)
if not result["OK"]:
print("Error:", result["Message"])
DIRACExit(-1)
jobs += result["Value"]
if len(args) < 1 and not jobs:
Script.showHelp(exitCode=1)
if len(args) > 0:
jobs += parseArguments(args)
result = dirac.getJobStatus(jobs)
if result["OK"]:
for job in result["Value"]:
print("JobID=" + str(job), end=" ")
for status in result["Value"][job].items():
print("%s=%s;" % status, end=" ")
print()
else:
exitCode = 2
print("ERROR: %s" % result["Message"])
DIRACExit(exitCode)
def main():
Script.registerSwitch("f:", "File=",
"Get output for jobs with IDs from the file")
Script.registerSwitch("g:", "JobGroup=",
"Get output for jobs in the given group")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["JobID: DIRAC Job ID"], mandatory=False)
sws, args = Script.parseCommandLine(ignoreErrors=True)
import DIRAC
from DIRAC.Interfaces.API.Dirac import Dirac, parseArguments
from DIRAC.Core.Utilities.Time import toString, date, day
dirac = Dirac()
jobs = []
for sw, value in sws:
if sw.lower() in ("f", "file"):
if os.path.exists(value):
jFile = open(value)
jobs += jFile.read().split()
jFile.close()
elif sw.lower() in ("g", "jobgroup"):
group = value
jobDate = toString(date() - 30 * day)
result = dirac.selectJobs(jobGroup=value, date=jobDate)
if not result["OK"]:
if "No jobs selected" not in result["Message"]:
print("Error:", result["Message"])
DIRAC.exit(-1)
else:
jobs += result["Value"]
for arg in parseArguments(args):
jobs.append(arg)
if not jobs:
print("Warning: no jobs selected")
Script.showHelp()
DIRAC.exit(0)
result = dirac.deleteJob(jobs)
if result["OK"]:
print("Deleted jobs %s" % ",".join([str(j) for j in result["Value"]]))
exitCode = 0
else:
print(result["Message"])
exitCode = 2
DIRAC.exit(exitCode)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"LFN: Logical File Name or file containing LFNs")
Script.registerArgument("Dest: Valid DIRAC SE")
Script.registerArgument("Source: Valid DIRAC SE",
default="",
mandatory=False)
Script.registerArgument("Cache: Local directory to be used as cache",
default="",
mandatory=False)
_, args = Script.parseCommandLine(ignoreErrors=True)
if len(args) > 4:
Script.showHelp(exitCode=1)
lfn, seName, sourceSE, localCache = Script.getPositionalArgs(group=True)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
try:
f = open(lfn, "r")
lfns = f.read().splitlines()
f.close()
except Exception:
lfns = [lfn]
finalResult = {"Failed": [], "Successful": []}
for lfn in lfns:
result = dirac.replicateFile(lfn,
seName,
sourceSE,
localCache,
printOutput=True)
if not result["OK"]:
finalResult["Failed"].append(lfn)
print("ERROR %s" % (result["Message"]))
exitCode = 2
else:
finalResult["Successful"].append(lfn)
if len(lfns) > 1:
print(finalResult)
DIRAC.exit(exitCode)
def main():
global listOfFailedFiles
Script.registerSwitch("D", "sync", "Make target directory identical to source")
Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
(
"LFN: Logical File Name (Path to directory)",
"Path: Local path to the file (Path to directory)",
)
)
Script.registerArgument(
(
"Path: Local path to the file (Path to directory)",
"LFN: Logical File Name (Path to directory)",
)
)
Script.registerArgument(" SE: DIRAC Storage Element", mandatory=False)
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) > 3:
Script.showHelp()
sync = False
parallel = 1
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "s" or switch[0].lower() == "sync":
sync = True
if switch[0].lower() == "j" or switch[0].lower() == "parallel":
parallel = int(switch[1])
listOfFailedFiles = Manager().list()
# This is the execution
returnValue = run(args, sync, parallel)
if listOfFailedFiles:
gLogger.error("Some file operations failed:\n\t", "\n\t".join(listOfFailedFiles))
DIRAC.exit(1)
if not returnValue["OK"]:
gLogger.fatal(returnValue["Message"])
DIRAC.exit(1)
else:
gLogger.notice(returnValue["Value"])
DIRAC.exit(0)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument((
"URL: URL of the service to ping (instead of System and Service)",
"System/Service: Full component name (ie: WorkloadManagement/Matcher)",
"System: Name of the DIRAC system (ie: WorkloadManagement)",
))
Script.registerArgument(
" Service: Name of the DIRAC service (ie: Matcher)",
mandatory=False)
_, args = Script.parseCommandLine(ignoreErrors=True)
system = None
service = None
url = None
if len(args) == 1:
# it is a URL
if args[0].startswith("dips://"):
url = args[0]
# It is System/Service
else:
sys_serv = args[0].split("/")
if len(sys_serv) != 2:
Script.showHelp(exitCode=1)
else:
system, service = sys_serv
elif len(args) == 2:
system, service = args[0], args[1]
else:
Script.showHelp(exitCode=1)
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
result = dirac.pingService(system, service, printOutput=True, url=url)
if not result:
print("ERROR: Null result from ping()")
exitCode = 2
elif not result["OK"]:
print("ERROR: ", result["Message"])
exitCode = 2
DIRAC.exit(exitCode)
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 2:
Script.showHelp()
seName = args[1]
fileName = args[0]
import os
from DIRAC import exit as DIRACExit, gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.StorageManagementSystem.Client.StorageManagerClient import StorageManagerClient
stageLfns = {}
if os.path.exists(fileName):
try:
lfnFile = open(fileName)
lfns = [k.strip() for k in lfnFile.readlines()]
lfnFile.close()
except Exception:
gLogger.exception("Can not open file", fileName)
DIRACExit(-1)
else:
lfns = args[: len(args) - 1]
stageLfns[seName] = lfns
stagerClient = StorageManagerClient()
res = stagerClient.setRequest(
stageLfns, "WorkloadManagement", "updateJobFromStager@WorkloadManagement/JobStateUpdate", 0
) # fake JobID = 0
if not res["OK"]:
gLogger.error(res["Message"])
DIRACExit(-1)
else:
gLogger.notice("Stage request submitted for LFNs:\n %s" % lfns)
gLogger.notice("SE= %s" % seName)
gLogger.notice("You can check their status and progress with dirac-stager-monitor-file <LFN> <SE>")
DIRACExit()
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("transID: transformation ID")
_, args = Script.parseCommandLine()
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if len(args) != 1:
Script.showHelp(exitCode=1)
tc = TransformationClient()
res = tc.getTransformationFiles({"TransformationID": args[0]})
if not res["OK"]:
DIRAC.gLogger.error(res["Message"])
DIRAC.exit(2)
for transfile in res["Value"]:
DIRAC.gLogger.notice(transfile["LFN"])
def _setSwitches(self):
Script.registerSwitch("S:", "system=", "Systems to check, by default all of them are checked", self._setSystems)
Script.registerSwitch("M", "modified", "Show entries which differ from the default", self._setShowModified)
Script.registerSwitch("A", "added", "Show entries which do not exist in ConfigTemplate", self._setShowAdded)
Script.registerSwitch(
"U",
"missingSection",
"Show sections which do not exist in the current configuration",
self._setShowMissingSections,
)
Script.registerSwitch(
"O",
"missingOption",
"Show options which do not exist in the current configuration",
self._setShowMissingOptions,
)
Script.parseCommandLine(ignoreErrors=True)
if not any([self.showModified, self.showAdded, self.showMissingSections, self.showMissingOptions]):
LOG.error("\nERROR: Set at least one of the flags M A U O")
Script.showHelp()
def main():
cliParams = Params()
Script.disableCS()
Script.registerSwitch(
"e", "exitOnError",
"flag to exit on error of any component installation",
cliParams.setExitOnError)
Script.addDefaultOptionValue("/DIRAC/Security/UseServerCertificate", "yes")
Script.addDefaultOptionValue("LogLevel", "INFO")
Script.parseCommandLine()
args = Script.getExtraCLICFGFiles()
if len(args) > 1:
Script.showHelp(exitCode=1)
cfg = None
if len(args):
cfg = args[0]
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.exitOnError = cliParams.exitOnError
result = gComponentInstaller.setupSite(Script.localCfg, cfg)
if not result["OK"]:
print("ERROR:", result["Message"])
exit(-1)
result = gComponentInstaller.getStartupComponentStatus([])
if not result["OK"]:
print("ERROR:", result["Message"])
exit(-1)
print("\nStatus of installed components:\n")
result = gComponentInstaller.printStartupStatus(result["Value"])
if not result["OK"]:
print("ERROR:", result["Message"])
exit(-1)
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
for key in ("status", "se", "limit"):
if key not in switches:
subLogger.warn(
"You're not using switch --%s, query may take long!" % key)
if "status" in switches and switches["status"] not in (
"New",
"Offline",
"Waiting",
"Failed",
"StageSubmitted",
"Staged",
):
subLogger.error(
'Found "%s" as Status value. Incorrect value used!' %
switches["status"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def main():
Script.disableCS()
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
" System: Name of the system for the component (default *: all)", mandatory=False, default="*"
)
Script.registerArgument(
(
"Service: Name of the particular component (default *: all)",
"Agent: Name of the particular component (default *: all)",
),
mandatory=False,
default="*",
)
_, args = Script.parseCommandLine()
system, component = Script.getPositionalArgs(group=True)
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
__RCSID__ = "$Id$"
if len(args) > 2:
Script.showHelp(exitCode=1)
if len(args) > 0:
system = args[0]
if system != "*":
if len(args) > 1:
component = args[1]
#
gComponentInstaller.exitOnError = True
#
result = gComponentInstaller.getStartupComponentStatus([system, component])
if not result["OK"]:
print("ERROR:", result["Message"])
exit(-1)
gComponentInstaller.printStartupStatus(result["Value"])
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
switches.setdefault("statusType", None)
switches.setdefault("VO", None)
for key in ("element", "name", "status", "reason"):
if key not in switches:
subLogger.error("%s Switch missing" % key)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
if not switches["element"] in ("Site", "Resource", "Node"):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
statuses = StateMachine.RSSMachine(None).getStates()
if not switches["status"] in statuses:
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
#!/usr/bin/env python
""" Drop DBs from the MySQL server
"""
from __future__ import print_function
from __future__ import absolute_import
from __future__ import division
from DIRAC.Core.Utilities.DIRACScript import DIRACScript as Script
Script.setUsageMessage("\n".join([
__doc__.split("\n")[1],
"Usage:",
" %s [options] ... DB ..." % Script.scriptName,
"Arguments:",
" DB: Name of the Database (mandatory)",
]))
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) < 1:
Script.showHelp(exitCode=1)
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.getMySQLPasswords()
for db in args:
print(gComponentInstaller.execMySQL("DROP DATABASE IF EXISTS %s" % db))
def addProperty(arg):
global groupProperties
if not arg:
Script.showHelp(exitCode=1)
if arg not in groupProperties:
groupProperties.append(arg)
def addUserName(arg):
global userNames
if not arg:
Script.showHelp(exitCode=1)
if arg not in userNames:
userNames.append(arg)
def setGroupName(arg):
global groupName
if groupName or not arg:
Script.showHelp(exitCode=1)
groupName = arg
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" SE: StorageElement|All")
Script.registerArgument(["LFN: LFN or file containing a List of LFNs"])
Script.parseCommandLine(ignoreErrors=False)
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
args = Script.getPositionalArgs()
targetSE = args.pop(0)
lfns = []
for inputFileName in args:
if os.path.exists(inputFileName):
with open(inputFileName, "r") as inputFile:
string = inputFile.read()
lfns.extend([lfn.strip() for lfn in string.splitlines()])
else:
lfns.append(inputFileName)
from DIRAC.Resources.Storage.StorageElement import StorageElement
import DIRAC
# Check is provided SE is OK
if targetSE != "All":
se = StorageElement(targetSE)
if not se.valid:
print(se.errorReason)
print()
Script.showHelp()
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.private.RequestValidator import RequestValidator
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
reqClient = ReqClient()
fc = FileCatalog()
requestOperation = "RemoveReplica"
if targetSE == "All":
requestOperation = "RemoveFile"
for lfnList in breakListIntoChunks(lfns, 100):
oRequest = Request()
requestName = "%s_%s" % (
md5(repr(time.time()).encode()).hexdigest()[:16],
md5(repr(time.time()).encode()).hexdigest()[:16],
)
oRequest.RequestName = requestName
oOperation = Operation()
oOperation.Type = requestOperation
oOperation.TargetSE = targetSE
res = fc.getFileMetadata(lfnList)
if not res["OK"]:
print("Can't get file metadata: %s" % res["Message"])
DIRAC.exit(1)
if res["Value"]["Failed"]:
print(
"Could not get the file metadata of the following, so skipping them:"
)
for fFile in res["Value"]["Failed"]:
print(fFile)
lfnMetadata = res["Value"]["Successful"]
for lfn in lfnMetadata:
rarFile = File()
rarFile.LFN = lfn
rarFile.Size = lfnMetadata[lfn]["Size"]
rarFile.Checksum = lfnMetadata[lfn]["Checksum"]
rarFile.GUID = lfnMetadata[lfn]["GUID"]
rarFile.ChecksumType = "ADLER32"
oOperation.addFile(rarFile)
oRequest.addOperation(oOperation)
isValid = RequestValidator().validate(oRequest)
if not isValid["OK"]:
print("Request is not valid: ", isValid["Message"])
DIRAC.exit(1)
result = reqClient.putRequest(oRequest)
if result["OK"]:
print("Request %d Submitted" % result["Value"])
else:
print("Failed to submit Request: ", result["Message"])
def main():
global overwrite
global specialOptions
global module
global specialOptions
from DIRAC.FrameworkSystem.Client.ComponentInstaller import gComponentInstaller
gComponentInstaller.exitOnError = True
Script.registerSwitch("w", "overwrite",
"Overwrite the configuration in the global CS",
setOverwrite)
Script.registerSwitch("m:", "module=",
"Python module name for the component code",
setModule)
Script.registerSwitch("p:", "parameter=", "Special component option ",
setSpecialOption)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument((
"System/Component: Full component name (ie: WorkloadManagement/Matcher)",
"System: Name of the DIRAC system (ie: WorkloadManagement)",
))
Script.registerArgument(
" Component: Name of the DIRAC service (ie: Matcher)",
mandatory=False)
Script.parseCommandLine()
args = Script.getPositionalArgs()
if len(args) == 1:
args = args[0].split("/")
if len(args) != 2:
Script.showHelp(exitCode=1)
system = args[0]
component = args[1]
compOrMod = module or component
result = gComponentInstaller.getSoftwareComponents(extensionsByPriority())
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
availableComponents = result["Value"]
for compType in availableComponents:
if system in availableComponents[
compType] and compOrMod in availableComponents[compType][
system]:
cType = compType[:-1].lower()
break
else:
gLogger.error("Component %s/%s is not available for installation" %
(system, component))
DIRACexit(1)
if module:
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
cType,
system,
module,
extensionsByPriority(),
overwrite=overwrite)
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
cType,
system,
component,
extensionsByPriority(),
specialOptions=specialOptions,
overwrite=overwrite,
addDefaultOptions=False,
)
else:
result = gComponentInstaller.addDefaultOptionsToCS(
gConfig,
cType,
system,
component,
extensionsByPriority(),
specialOptions=specialOptions,
overwrite=overwrite,
)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
result = gComponentInstaller.installComponent(cType, system, component,
extensionsByPriority(),
module)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
gLogger.notice(
"Successfully installed component %s in %s system, now setting it up" %
(component, system))
result = gComponentInstaller.setupComponent(cType, system, component,
extensionsByPriority(), module)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
if component == "ComponentMonitoring":
result = MonitoringUtilities.monitorInstallation(
"DB", system, "InstalledComponentsDB")
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
result = MonitoringUtilities.monitorInstallation(cType, system, component,
module)
if not result["OK"]:
gLogger.error(result["Message"])
DIRACexit(1)
gLogger.notice("Successfully completed the installation of %s/%s" %
(system, component))
DIRACexit()
def main():
days = 0
months = 0
years = 0
wildcard = None
baseDir = ""
emptyDirsFlag = False
Script.registerSwitch("D:", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"M:", "Months=",
"Match files older than number of months [%s]" % months)
Script.registerSwitch(
"Y:", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("w:", "Wildcard=",
"Wildcard for matching filenames [All]")
Script.registerSwitch(
"b:", "BaseDir=",
"Base directory to begin search (default /[vo]/user/[initial]/[username])"
)
Script.registerSwitch("e", "EmptyDirs",
"Create a list of empty directories")
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0] == "D" or switch[0].lower() == "days":
days = int(switch[1])
if switch[0] == "M" or switch[0].lower() == "months":
months = int(switch[1])
if switch[0] == "Y" or switch[0].lower() == "years":
years = int(switch[1])
if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
wildcard = "*" + switch[1]
if switch[0].lower() == "b" or switch[0].lower() == "basedir":
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from datetime import datetime, timedelta
import sys
import os
import time
import fnmatch
fc = FileCatalog()
def isOlderThan(cTimeStruct, days):
timeDelta = timedelta(days=days)
maxCTime = datetime.utcnow() - timeDelta
if cTimeStruct < maxCTime:
return True
return False
withMetadata = False
if days or months or years:
withMetadata = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res["OK"]:
gLogger.error("Failed to get client proxy information.",
res["Message"])
DIRAC.exit(2)
proxyInfo = res["Value"]
if proxyInfo["secondsLeft"] == 0:
gLogger.error("Proxy expired")
DIRAC.exit(2)
username = proxyInfo["username"]
vo = ""
if "group" in proxyInfo:
vo = getVOForGroup(proxyInfo["group"])
if not baseDir:
if not vo:
gLogger.error("Could not determine VO")
Script.showHelp()
baseDir = "/%s/user/%s/%s" % (vo, username[0], username)
baseDir = baseDir.rstrip("/")
gLogger.notice("Will search for files in %s%s" %
(baseDir, (" matching %s" % wildcard) if wildcard else ""))
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len(activeDirs) > 0:
currentDir = activeDirs.pop()
res = fc.listDirectory(currentDir, withMetadata, timeout=360)
if not res["OK"]:
gLogger.error("Error retrieving directory contents",
"%s %s" % (currentDir, res["Message"]))
elif currentDir in res["Value"]["Failed"]:
gLogger.error(
"Error retrieving directory contents",
"%s %s" % (currentDir, res["Value"]["Failed"][currentDir]))
else:
dirContents = res["Value"]["Successful"][currentDir]
subdirs = dirContents["SubDirs"]
files = dirContents["Files"]
if not subdirs and not files:
emptyDirs.append(currentDir)
gLogger.notice("%s: empty directory" % currentDir)
else:
for subdir in sorted(subdirs, reverse=True):
if (not withMetadata) or isOlderThan(
subdirs[subdir]["CreationDate"], totalDays):
activeDirs.append(subdir)
for filename in sorted(files):
fileOK = False
if (not withMetadata) or isOlderThan(
files[filename]["MetaData"]["CreationDate"],
totalDays):
if wildcard is None or fnmatch.fnmatch(
filename, wildcard):
fileOK = True
if not fileOK:
files.pop(filename)
allFiles += sorted(files)
if len(files) or len(subdirs):
gLogger.notice(
"%s: %d files%s, %d sub-directories" %
(currentDir, len(files), " matching"
if withMetadata or wildcard else "", len(subdirs)))
outputFileName = "%s.lfns" % baseDir.replace("/%s" % vo,
"%s" % vo).replace("/", "-")
outputFile = open(outputFileName, "w")
for lfn in sorted(allFiles):
outputFile.write(lfn + "\n")
outputFile.close()
gLogger.notice("%d matched files have been put in %s" %
(len(allFiles), outputFileName))
if emptyDirsFlag:
outputFileName = "%s.emptydirs" % baseDir.replace(
"/%s" % vo, "%s" % vo).replace("/", "-")
outputFile = open(outputFileName, "w")
for dir in sorted(emptyDirs):
outputFile.write(dir + "\n")
outputFile.close()
gLogger.notice("%d empty directories have been put in %s" %
(len(emptyDirs), outputFileName))
DIRAC.exit(0)