def main():
Script.registerSwitch("e", "extended", "Show extended info")
Script.parseCommandLine(ignoreErrors=True)
from DIRAC import exit as DIRACExit
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
extendedInfo = False
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("e", "extended"):
extendedInfo = True
if not extendedInfo:
result = diracAdmin.csListHosts()
for host in result["Value"]:
print(" %s" % host)
else:
result = diracAdmin.csDescribeHosts()
print(diracAdmin.pPrint.pformat(result["Value"]))
for error in errorList:
print("ERROR %s: %s" % error)
DIRACExit(exitCode)
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ":" if short else "",
longOption + "=", doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose("Found switch %r with value %r" %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches["DryRun"] = not self.switches.get("Execute", False)
self.switches["SourceSE"] = self.switches.get("SourceSE",
"").split(",")
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("element", None)
switches.setdefault("defaultStatus", "Banned")
if not switches["element"] in ("all", "Site", "Resource", "Node", None):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def main():
Script.registerSwitch(
"v:",
"vo=",
"Location of pilot version in CS /Operations/<vo>/Pilot/Version"
" (default value specified in CS under /DIRAC/DefaultSetup)",
)
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("version: pilot version you want to update to")
Script.parseCommandLine(ignoreErrors=False)
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
version = Script.getPositionalArgs(group=True)
vo = None
for switch in Script.getUnprocessedSwitches():
if switch[0] == "v" or switch[0] == "vo":
vo = switch[1]
from DIRAC import S_OK, S_ERROR
from DIRAC import gConfig, gLogger
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
def updatePilot(version, vo):
"""
Update in the CS the pilot version used,
If only one version present in CS it's overwritten.
If two versions present, the new one is added and the last removed
:param version: version vArBpC of pilot you want to use
:param vo: Location of pilot version in CS /Operations/<vo>/Pilot/Version
"""
setup = vo
if not vo:
setup = gConfig.getValue("/DIRAC/DefaultSetup")
if not setup:
return S_ERROR("No value set for /DIRAC/DefaultSetup in CS")
pilotVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup, [])
if not pilotVersion:
return S_ERROR("No pilot version set under Operations/%s/Pilot/Version in CS" % setup)
pilotVersion.pop()
pilotVersion.insert(0, version)
api = CSAPI()
api.setOption("Operations/%s/Pilot/Version" % setup, ", ".join(pilotVersion))
result = api.commit()
if not result["OK"]:
gLogger.fatal("Could not commit new version of pilot!")
return result
newVersion = gConfig.getValue("Operations/%s/Pilot/Version" % setup)
return S_OK("New version of pilot set to %s" % newVersion)
result = updatePilot(version, vo)
if not result["OK"]:
gLogger.fatal(result["Message"])
DIRAC.exit(1)
gLogger.notice(result["Value"])
DIRAC.exit(0)
def parseSwitchesAndPositionalArguments():
"""
Parse switches and positional arguments given to the script
"""
# Parse the command line and initialize DIRAC
Script.parseCommandLine(ignoreErrors=False)
# Get arguments
allArgs = Script.getPositionalArgs()
gLogger.debug("All arguments: %s" % ", ".join(allArgs))
# Get unprocessed switches
switches = dict(Script.getUnprocessedSwitches())
gLogger.debug("The switches used are:")
map(gLogger.debug, switches.iteritems())
# Get grouped positional arguments
repType, user, services = Script.getPositionalArgs(group=True)
gLogger.debug("The positional arguments are:")
gLogger.debug("Report type:", repType)
gLogger.debug("Name or DN:", user)
gLogger.debug("Services:", services)
return switches, repType, user, services
def processScriptSwitches():
global vo, dry, doCEs, hostURL, onecore
Script.registerSwitch("V:", "vo=", "Virtual Organization")
Script.registerSwitch("D", "dry", "Dry run")
Script.registerSwitch("C", "ce", "Process Computing Elements")
Script.registerSwitch("H:", "host=", "use this url for information querying")
Script.registerSwitch(
"", "onecore", "Add Single Core Queues for each MultiCore Queue, set RequiredTag for those Queues"
)
Script.parseCommandLine(ignoreErrors=True)
vo = ""
dry = False
doCEs = False
hostURL = None
onecore = False
for sw in Script.getUnprocessedSwitches():
if sw[0] in ("V", "vo"):
vo = sw[1]
if sw[0] in ("D", "dry"):
dry = True
if sw[0] in ("C", "ce"):
doCEs = True
if sw[0] in ("H", "host"):
hostURL = sw[1]
if sw[0] in ("onecore",):
onecore = True
def main():
Script.registerSwitch("", "Full", " Print full list of requests")
Script.parseCommandLine()
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
fullPrint = False
for switch in Script.getUnprocessedSwitches():
if switch[0] == "Full":
fullPrint = True
reqClient = ReqClient()
for server, rpcClient in reqClient.requestProxies().items():
DIRAC.gLogger.always("Checking request cache at %s" % server)
reqCache = rpcClient.listCacheDir()
if not reqCache["OK"]:
DIRAC.gLogger.error("Cannot list request cache", reqCache)
continue
reqCache = reqCache["Value"]
if not reqCache:
DIRAC.gLogger.always("No request in cache")
else:
if fullPrint:
DIRAC.gLogger.always("List of requests", reqCache)
else:
DIRAC.gLogger.always("Number of requests in the cache",
len(reqCache))
DIRAC.exit(0)
def registerSwitchesAndParseCommandLine(self):
"""Register the default plus additional parameters and parse options.
:param list options: list of three tuple for options to add to the script
:param list flags: list of three tuple for flags to add to the script
:param str opName
"""
for short, longOption, doc in self.options:
Script.registerSwitch(short + ":" if short else "",
longOption + "=", doc)
for short, longOption, doc in self.flags:
Script.registerSwitch(short, longOption, doc)
self.switches[longOption] = False
Script.parseCommandLine()
if Script.getPositionalArgs():
Script.showHelp(exitCode=1)
ops = Operations()
if not ops.getValue("DataManagement/ArchiveFiles/Enabled", False):
sLog.error(
'The "ArchiveFiles" operation is not enabled, contact your administrator!'
)
DIRAC.exit(1)
for _short, longOption, _doc in self.options:
defaultValue = ops.getValue(
"DataManagement/ArchiveFiles/%s" % longOption, None)
if defaultValue:
sLog.verbose(
"Found default value in the CS for %r with value %r" %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for _short, longOption, _doc in self.flags:
defaultValue = ops.getValue(
"DataManagement/ArchiveFiles/%s" % longOption, False)
if defaultValue:
sLog.verbose(
"Found default value in the CS for %r with value %r" %
(longOption, defaultValue))
self.switches[longOption] = defaultValue
for switch in Script.getUnprocessedSwitches():
for short, longOption, doc in self.options:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
sLog.verbose("Found switch %r with value %r" %
(longOption, switch[1]))
self.switches[longOption] = switch[1]
break
for short, longOption, doc in self.flags:
if switch[0] == short or switch[0].lower() == longOption.lower(
):
self.switches[longOption] = True
break
self.checkSwitches()
self.switches["DryRun"] = not self.switches.get("Execute", False)
self.switches["SourceSE"] = self.switches.get("SourceSE",
"").split(",")
def main():
Script.registerSwitch("N:", "NumberOfProcessors=", "Run n parallel copies of the benchmark")
Script.registerSwitch("U", "Update", "Update dirac.cfg with the resulting value")
Script.registerSwitch("R:", "Reconfig=", "Update given configuration file with the resulting value")
Script.parseCommandLine(ignoreErrors=True)
update = False
configFile = None
numberOfProcessors = 0
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("U", "Update"):
update = True
elif unprocSw[0] in ("R", "Reconfig"):
configFile = unprocSw[1]
elif unprocSw[0] in ("N", "NumberOfProcessors"):
try:
numberOfProcessors = int(unprocSw[1])
except ValueError:
gLogger.warn("Cannot make benchmark measurements: NumberOfProcessors is not a number")
# if numberOfProcessors has not been provided, try to get it from the configuration
if not numberOfProcessors:
numberOfProcessors = gConfig.getValue("/Resources/Computing/CEDefaults/NumberOfProcessors", 1)
gLogger.info("Computing benchmark measurements on", "%d processor(s)..." % numberOfProcessors)
# we want to get the logs coming from db12
gLogger.enableLogsFromExternalLibs()
# multiprocessor allocations generally have a CPU Power lower than single core one.
# in order to avoid having wrong estimations, we run multiple copies of the benchmark simultaneously
result = multipleDiracBenchmark(numberOfProcessors)
if result is None:
gLogger.error("Cannot make benchmark measurements")
DIRAC.exit(1)
# we take a conservative approach and use the minimum value returned as the CPU Power
db12Result = min(result["raw"])
# because hardware is continuously evolving, original benchmark scores might need a correction
corr = Operations().getValue("JobScheduling/CPUNormalizationCorrection", 1.0)
gLogger.info("Applying a correction on the CPU power:", corr)
cpuPower = round(db12Result / corr, 1)
gLogger.notice("Estimated CPU power is %.1f HS06" % cpuPower)
if update:
gConfig.setOptionValue("/LocalSite/CPUNormalizationFactor", cpuPower)
if configFile:
gConfig.dumpLocalCFGToFile(configFile)
else:
gConfig.dumpLocalCFGToFile(gConfig.diracConfigFilePath)
DIRAC.exit()
def main():
Script.registerSwitch("p:", "property=",
"Add property to the user <name>=<value>")
Script.registerSwitch("f", "force", "create the user if it doesn't exist")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" user: User name")
Script.registerArgument(" DN: DN of the User")
Script.registerArgument(["group: Add the user to the group"])
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
forceCreation = False
errorList = []
userProps = {}
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("f", "force"):
forceCreation = True
elif unprocSw[0] in ("p", "property"):
prop = unprocSw[1]
pl = prop.split("=")
if len(pl) < 2:
errorList.append((
"in arguments",
"Property %s has to include a '=' to separate name from value"
% prop))
exitCode = 255
else:
pName = pl[0]
pValue = "=".join(pl[1:])
print("Setting property %s to %s" % (pName, pValue))
userProps[pName] = pValue
userName, userProps["DN"], userProps["Groups"] = Script.getPositionalArgs(
group=True)
if not diracAdmin.csModifyUser(
userName, userProps, createIfNonExistant=forceCreation):
errorList.append(("modify user", "Cannot modify user %s" % userName))
exitCode = 255
else:
result = diracAdmin.csCommitChanges()
if not result["OK"]:
errorList.append(("commit", result["Message"]))
exitCode = 255
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def main():
host = None
Script.registerSwitch("H:", "host=", " Target host")
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "h" or switch[0].lower() == "host":
host = switch[1]
from DIRAC.FrameworkSystem.Client.SystemAdministratorClientCLI import SystemAdministratorClientCLI
cli = SystemAdministratorClientCLI(host)
cli.cmdloop()
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if len(args) < 3:
error(
"Missing all mandatory 'query', 'element', 'tableType' arguments")
elif args[0].lower() not in ("select", "add", "modify", "delete"):
error("Incorrect 'query' argument")
elif args[1].lower() not in ("site", "resource", "component", "node"):
error("Incorrect 'element' argument")
elif args[2].lower() not in ("status", "log", "history"):
error("Incorrect 'tableType' argument")
else:
query = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("name", None)
switches.setdefault("statusType", None)
switches.setdefault("status", None)
switches.setdefault("elementType", None)
switches.setdefault("reason", None)
switches.setdefault("lastCheckTime", None)
switches.setdefault("tokenOwner", None)
switches.setdefault("VO", None)
if "status" in switches and switches["status"] is not None:
switches["status"] = switches["status"].title()
if switches["status"] not in ("Active", "Probing", "Degraded",
"Banned", "Unknown", "Error"):
error("'%s' is an invalid argument for switch 'status'" %
switches["status"])
# when it's a add/modify query and status/reason/statusType are not specified
# then some specific defaults are set up
if query == "add" or query == "modify":
if "status" not in switches or switches["status"] is None:
switches["status"] = "Unknown"
if "reason" not in switches or switches["reason"] is None:
switches["reason"] = "Unknown reason"
if "statusType" not in switches or switches["statusType"] is None:
switches["statusType"] = "all"
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return args, switches
def main():
Script.registerSwitch("e", "extended", "Show extended info")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["Group: Only users from this group (default: all)"], default=["all"], mandatory=False)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs(group=True)
import DIRAC
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
extendedInfo = False
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("e", "extended"):
extendedInfo = True
def printUsersInGroup(group=False):
result = diracAdmin.csListUsers(group)
if result["OK"]:
if group:
print("Users in group %s:" % group)
else:
print("All users registered:")
for username in result["Value"]:
print(" %s" % username)
def describeUsersInGroup(group=False):
result = diracAdmin.csListUsers(group)
if result["OK"]:
if group:
print("Users in group %s:" % group)
else:
print("All users registered:")
result = diracAdmin.csDescribeUsers(result["Value"])
print(diracAdmin.pPrint.pformat(result["Value"]))
for group in args:
if "all" in args:
group = False
if not extendedInfo:
printUsersInGroup(group)
else:
describeUsersInGroup(group)
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def main():
Script.registerSwitch("C:", "CPUNormalizationFactor=",
"CPUNormalizationFactor, in case it is known")
Script.parseCommandLine(ignoreErrors=True)
CPUNormalizationFactor = 0.0
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("C", "CPUNormalizationFactor"):
CPUNormalizationFactor = float(unprocSw[1])
from DIRAC.WorkloadManagementSystem.Client.CPUNormalization import getCPUTime
cpuTime = getCPUTime(CPUNormalizationFactor)
# I hate this kind of output... PhC
print("CPU time left determined as", cpuTime)
DIRAC.exit(0)
def main():
global listOfFailedFiles
Script.registerSwitch("D", "sync", "Make target directory identical to source")
Script.registerSwitch("j:", "parallel=", "Multithreaded download and upload")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
(
"LFN: Logical File Name (Path to directory)",
"Path: Local path to the file (Path to directory)",
)
)
Script.registerArgument(
(
"Path: Local path to the file (Path to directory)",
"LFN: Logical File Name (Path to directory)",
)
)
Script.registerArgument(" SE: DIRAC Storage Element", mandatory=False)
Script.parseCommandLine(ignoreErrors=False)
args = Script.getPositionalArgs()
if len(args) > 3:
Script.showHelp()
sync = False
parallel = 1
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "s" or switch[0].lower() == "sync":
sync = True
if switch[0].lower() == "j" or switch[0].lower() == "parallel":
parallel = int(switch[1])
listOfFailedFiles = Manager().list()
# This is the execution
returnValue = run(args, sync, parallel)
if listOfFailedFiles:
gLogger.error("Some file operations failed:\n\t", "\n\t".join(listOfFailedFiles))
DIRAC.exit(1)
if not returnValue["OK"]:
gLogger.fatal(returnValue["Message"])
DIRAC.exit(1)
else:
gLogger.notice(returnValue["Value"])
DIRAC.exit(0)
def main():
Script.registerSwitch("t", "test", "Only test. Don't commit changes")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"UserCfg: Cfg FileName with Users as sections containing"
"DN, Groups, and other properties as options")
Script.parseCommandLine(ignoreErrors=True)
args = Script.getExtraCLICFGFiles()
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
diracAdmin = DiracAdmin()
exitCode = 0
testOnly = False
errorList = []
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("t", "test"):
testOnly = True
try:
usersCFG = CFG().loadFromFile(args[0])
except Exception as e:
errorList.append("file open",
"Can't parse file %s: %s" % (args[0], str(e)))
errorCode = 1
else:
if not diracAdmin.csSyncUsersWithCFG(usersCFG):
errorList.append(("modify users", "Cannot sync with %s" % args[0]))
exitCode = 255
if not exitCode and not testOnly:
result = diracAdmin.csCommitChanges()
if not result["OK"]:
errorList.append(("commit", result["Message"]))
exitCode = 255
for error in errorList:
print("ERROR %s: %s" % error)
DIRAC.exit(exitCode)
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
switches.setdefault("statusType", None)
switches.setdefault("VO", None)
for key in ("element", "name", "status", "reason"):
if key not in switches:
subLogger.error("%s Switch missing" % key)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
if not switches["element"] in ("Site", "Resource", "Node"):
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
statuses = StateMachine.RSSMachine(None).getStates()
if not switches["status"] in statuses:
subLogger.error("Found %s as element switch" % switches["element"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def main():
Script.registerSwitch("D:", "Download=",
"Defines data acquisition as DownloadInputData")
Script.registerSwitch("P:", "Protocol=",
"Defines data acquisition as InputDataByProtocol")
Script.parseCommandLine(ignoreErrors=False)
_downloadinputdata = False
_jobID = None
for switch in Script.getUnprocessedSwitches():
if switch[0] in ("D", "Download"):
_downloadinputdata = True
_jobID = switch[1]
if switch[0] in ("I", "Protocol"):
_downloadinputdata = False
_jobID = switch[1]
from DIRAC.ConfigurationSystem.Client.Helpers.CSGlobals import Extensions
ext = Extensions()
_vo = ext.getCSExtensions()[0]
_diracPath = Extensions().getExtensionPath("DIRAC")
_dir = os.path.expanduser("~") + os.path.sep
try:
_path = __runSystemDefaults(_jobID, _vo)
__downloadJobDescriptionXML(_jobID, _path)
__modifyJobDescription(_jobID, _path, _downloadinputdata)
__downloadPilotScripts(_path, _diracPath)
__configurePilot(_path, _vo)
__runJobLocally(_jobID, _path, _vo)
finally:
os.chdir(_dir)
os.rename(_dir + ".dirac.cfg.old", _dir + ".dirac.cfg")
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if args:
subLogger.error(
"Found the following positional args '%s', but we only accept switches"
% args)
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
switches = dict(Script.getUnprocessedSwitches())
for key in ("status", "se", "limit"):
if key not in switches:
subLogger.warn(
"You're not using switch --%s, query may take long!" % key)
if "status" in switches and switches["status"] not in (
"New",
"Offline",
"Waiting",
"Failed",
"StageSubmitted",
"Staged",
):
subLogger.error(
'Found "%s" as Status value. Incorrect value used!' %
switches["status"])
subLogger.error("Please, check documentation below")
Script.showHelp(exitCode=1)
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return switches
def parseSwitches():
"""
Parses the arguments passed by the user
"""
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
if not args:
error("Missing mandatory 'query' argument")
elif not args[0].lower() in ("select", "add", "delete"):
error("Missing mandatory argument")
else:
query = args[0].lower()
switches = dict(Script.getUnprocessedSwitches())
# Default values
switches.setdefault("downtimeID", None)
switches.setdefault("element", None)
switches.setdefault("name", None)
switches.setdefault("startDate", None)
switches.setdefault("endDate", None)
switches.setdefault("severity", None)
switches.setdefault("description", None)
switches.setdefault("link", None)
if query in ("add", "delete") and switches["downtimeID"] is None:
error("'downtimeID' switch is mandatory for '%s' but found missing" % query)
if query in ("add", "delete") and "ongoing" in switches:
error("'ongoing' switch can be used only with 'select'")
subLogger.debug("The switches used are:")
map(subLogger.debug, switches.items())
return (args, switches)
def main():
site = "BOINC.World.org"
status = ["Running"]
minorStatus = None
workerNodes = None
since = None
date = "today"
full = False
until = None
batchIDs = None
Script.registerSwitch("", "Site=", " Select site (default: %s)" % site)
Script.registerSwitch("", "Status=",
" Select status (default: %s)" % status)
Script.registerSwitch("", "MinorStatus=", " Select minor status")
Script.registerSwitch("", "WorkerNode=", " Select WN")
Script.registerSwitch("", "BatchID=", " Select batch jobID")
Script.registerSwitch(
"", "Since=",
" Date since when to select jobs, or number of days (default: today)"
)
Script.registerSwitch("", "Date=",
" Specify the date (check for a full day)")
Script.registerSwitch(
"", "Full",
" Printout full list of job (default: False except if --WorkerNode)")
Script.parseCommandLine()
from DIRAC import gLogger
from DIRAC.Interfaces.API.Dirac import Dirac
from DIRAC.WorkloadManagementSystem.Client.JobMonitoringClient import JobMonitoringClient
switches = Script.getUnprocessedSwitches()
for switch in switches:
if switch[0] == "Site":
site = switch[1]
elif switch[0] == "MinorStatus":
minorStatus = switch[1]
elif switch[0] == "Status":
if switch[1].lower() == "all":
status = [None]
else:
status = switch[1].split(",")
elif switch[0] == "WorkerNode":
workerNodes = switch[1].split(",")
elif switch[0] == "BatchID":
try:
batchIDs = [int(id) for id in switch[1].split(",")]
except Exception:
gLogger.error("Invalid jobID", switch[1])
DIRAC.exit(1)
elif switch[0] == "Full":
full = True
elif switch[0] == "Date":
since = switch[1].split()[0]
until = str(
datetime.datetime.strptime(since, "%Y-%m-%d") +
datetime.timedelta(days=1)).split()[0]
elif switch[0] == "Since":
date = switch[1].lower()
if date == "today":
since = None
elif date == "yesterday":
since = 1
elif date == "ever":
since = 2 * 365
elif date.isdigit():
since = int(date)
date += " days"
else:
since = date
if isinstance(since, int):
since = str(datetime.datetime.now() -
datetime.timedelta(days=since)).split()[0]
if workerNodes or batchIDs:
# status = [None]
full = True
monitoring = JobMonitoringClient()
dirac = Dirac()
# Get jobs according to selection
jobs = set()
for stat in status:
res = dirac.selectJobs(site=site,
date=since,
status=stat,
minorStatus=minorStatus)
if not res["OK"]:
gLogger.error("Error selecting jobs", res["Message"])
DIRAC.exit(1)
allJobs = set(int(job) for job in res["Value"])
if until:
res = dirac.selectJobs(site=site, date=until, status=stat)
if not res["OK"]:
gLogger.error("Error selecting jobs", res["Message"])
DIRAC.exit(1)
allJobs -= set(int(job) for job in res["Value"])
jobs.update(allJobs)
if not jobs:
gLogger.always("No jobs found...")
DIRAC.exit(0)
# res = monitoring.getJobsSummary( jobs )
# print eval( res['Value'] )[jobs[0]]
allJobs = set()
result = {}
wnJobs = {}
gLogger.always("%d jobs found" % len(jobs))
# Get host name
for job in jobs:
res = monitoring.getJobParameter(job, "HostName")
node = res.get("Value", {}).get("HostName", "Unknown")
res = monitoring.getJobParameter(job, "LocalJobID")
batchID = res.get("Value", {}).get("LocalJobID", "Unknown")
if workerNodes:
if not [wn for wn in workerNodes if node.startswith(wn)]:
continue
allJobs.add(job)
if batchIDs:
if batchID not in batchIDs:
continue
allJobs.add(job)
if full or status == [None]:
allJobs.add(job)
result.setdefault(job, {})["Status"] = status
result[job]["Node"] = node
result[job]["LocalJobID"] = batchID
wnJobs[node] = wnJobs.setdefault(node, 0) + 1
# If necessary get jobs' status
statusCounters = {}
if allJobs:
allJobs = sorted(allJobs, reverse=True)
res = monitoring.getJobsStates(allJobs)
if not res["OK"]:
gLogger.error("Error getting job parameter", res["Message"])
else:
jobStates = res["Value"]
for job in allJobs:
stat = (
jobStates.get(job, {}).get("Status", "Unknown") + "; " +
jobStates.get(job, {}).get("MinorStatus", "Unknown") +
"; " +
jobStates.get(job, {}).get("ApplicationStatus", "Unknown"))
result[job]["Status"] = stat
statusCounters[stat] = statusCounters.setdefault(stat, 0) + 1
elif not workerNodes and not batchIDs:
allJobs = sorted(jobs, reverse=True)
# Print out result
if workerNodes or batchIDs:
gLogger.always("Found %d jobs at %s, WN %s (since %s):" %
(len(allJobs), site, workerNodes, date))
if allJobs:
gLogger.always("List of jobs:",
",".join([str(job) for job in allJobs]))
else:
if status == [None]:
gLogger.always("Found %d jobs at %s (since %s):" %
(len(allJobs), site, date))
for stat in sorted(statusCounters):
gLogger.always("%d jobs %s" % (statusCounters[stat], stat))
else:
gLogger.always("Found %d jobs %s at %s (since %s):" %
(len(allJobs), status, site, date))
gLogger.always(
"List of WNs:",
",".join([
"%s (%d)" % (node, wnJobs[node]) for node in sorted(
wnJobs,
key=cmp_to_key(lambda n1, n2: (wnJobs[n2] - wnJobs[n1])))
]),
)
if full:
if workerNodes or batchIDs:
nodeJobs = {}
for job in allJobs:
status = result[job]["Status"]
node = result[job]["Node"].split(".")[0]
jobID = result[job].get("LocalJobID")
nodeJobs.setdefault(node, []).append((jobID, job, status))
if not workerNodes:
workerNodes = sorted(nodeJobs)
for node in workerNodes:
for job in nodeJobs.get(node.split(".")[0], []):
gLogger.always("%s " % node + "(%s): %s - %s" % job)
else:
for job in allJobs:
status = result[job]["Status"]
node = result[job]["Node"]
jobID = result[job].get("LocalJobID")
gLogger.always("%s (%s): %s - %s" % (node, jobID, job, status))
def main():
days = 0
months = 0
years = 0
wildcard = None
baseDir = ""
emptyDirsFlag = False
Script.registerSwitch("D:", "Days=",
"Match files older than number of days [%s]" % days)
Script.registerSwitch(
"M:", "Months=",
"Match files older than number of months [%s]" % months)
Script.registerSwitch(
"Y:", "Years=", "Match files older than number of years [%s]" % years)
Script.registerSwitch("w:", "Wildcard=",
"Wildcard for matching filenames [All]")
Script.registerSwitch(
"b:", "BaseDir=",
"Base directory to begin search (default /[vo]/user/[initial]/[username])"
)
Script.registerSwitch("e", "EmptyDirs",
"Create a list of empty directories")
Script.parseCommandLine(ignoreErrors=False)
for switch in Script.getUnprocessedSwitches():
if switch[0] == "D" or switch[0].lower() == "days":
days = int(switch[1])
if switch[0] == "M" or switch[0].lower() == "months":
months = int(switch[1])
if switch[0] == "Y" or switch[0].lower() == "years":
years = int(switch[1])
if switch[0].lower() == "w" or switch[0].lower() == "wildcard":
wildcard = "*" + switch[1]
if switch[0].lower() == "b" or switch[0].lower() == "basedir":
baseDir = switch[1]
if switch[0].lower() == "e" or switch[0].lower() == "emptydirs":
emptyDirsFlag = True
import DIRAC
from DIRAC import gLogger
from DIRAC.ConfigurationSystem.Client.Helpers.Registry import getVOForGroup
from DIRAC.Core.Security.ProxyInfo import getProxyInfo
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from datetime import datetime, timedelta
import sys
import os
import time
import fnmatch
fc = FileCatalog()
def isOlderThan(cTimeStruct, days):
timeDelta = timedelta(days=days)
maxCTime = datetime.utcnow() - timeDelta
if cTimeStruct < maxCTime:
return True
return False
withMetadata = False
if days or months or years:
withMetadata = True
totalDays = 0
if years:
totalDays += 365 * years
if months:
totalDays += 30 * months
if days:
totalDays += days
res = getProxyInfo(False, False)
if not res["OK"]:
gLogger.error("Failed to get client proxy information.",
res["Message"])
DIRAC.exit(2)
proxyInfo = res["Value"]
if proxyInfo["secondsLeft"] == 0:
gLogger.error("Proxy expired")
DIRAC.exit(2)
username = proxyInfo["username"]
vo = ""
if "group" in proxyInfo:
vo = getVOForGroup(proxyInfo["group"])
if not baseDir:
if not vo:
gLogger.error("Could not determine VO")
Script.showHelp()
baseDir = "/%s/user/%s/%s" % (vo, username[0], username)
baseDir = baseDir.rstrip("/")
gLogger.notice("Will search for files in %s%s" %
(baseDir, (" matching %s" % wildcard) if wildcard else ""))
activeDirs = [baseDir]
allFiles = []
emptyDirs = []
while len(activeDirs) > 0:
currentDir = activeDirs.pop()
res = fc.listDirectory(currentDir, withMetadata, timeout=360)
if not res["OK"]:
gLogger.error("Error retrieving directory contents",
"%s %s" % (currentDir, res["Message"]))
elif currentDir in res["Value"]["Failed"]:
gLogger.error(
"Error retrieving directory contents",
"%s %s" % (currentDir, res["Value"]["Failed"][currentDir]))
else:
dirContents = res["Value"]["Successful"][currentDir]
subdirs = dirContents["SubDirs"]
files = dirContents["Files"]
if not subdirs and not files:
emptyDirs.append(currentDir)
gLogger.notice("%s: empty directory" % currentDir)
else:
for subdir in sorted(subdirs, reverse=True):
if (not withMetadata) or isOlderThan(
subdirs[subdir]["CreationDate"], totalDays):
activeDirs.append(subdir)
for filename in sorted(files):
fileOK = False
if (not withMetadata) or isOlderThan(
files[filename]["MetaData"]["CreationDate"],
totalDays):
if wildcard is None or fnmatch.fnmatch(
filename, wildcard):
fileOK = True
if not fileOK:
files.pop(filename)
allFiles += sorted(files)
if len(files) or len(subdirs):
gLogger.notice(
"%s: %d files%s, %d sub-directories" %
(currentDir, len(files), " matching"
if withMetadata or wildcard else "", len(subdirs)))
outputFileName = "%s.lfns" % baseDir.replace("/%s" % vo,
"%s" % vo).replace("/", "-")
outputFile = open(outputFileName, "w")
for lfn in sorted(allFiles):
outputFile.write(lfn + "\n")
outputFile.close()
gLogger.notice("%d matched files have been put in %s" %
(len(allFiles), outputFileName))
if emptyDirsFlag:
outputFileName = "%s.emptydirs" % baseDir.replace(
"/%s" % vo, "%s" % vo).replace("/", "-")
outputFile = open(outputFileName, "w")
for dir in sorted(emptyDirs):
outputFile.write(dir + "\n")
outputFile.close()
gLogger.notice("%d empty directories have been put in %s" %
(len(emptyDirs), outputFileName))
DIRAC.exit(0)
def main():
Script.registerSwitch("", "Status=", "Primary status")
Script.registerSwitch("", "MinorStatus=", "Secondary status")
Script.registerSwitch("", "ApplicationStatus=", "Application status")
Script.registerSwitch("", "Site=", "Execution site")
Script.registerSwitch("", "Owner=", "Owner (DIRAC nickname)")
Script.registerSwitch("", "JobGroup=",
"Select jobs for specified job group")
Script.registerSwitch(
"", "Date=",
"Date in YYYY-MM-DD format, if not specified default is today")
Script.registerSwitch("", "File=",
"File name,if not specified default is std.out ")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("String: string to search for")
_, args = Script.parseCommandLine(ignoreErrors=True)
# Default values
status = None
minorStatus = None
appStatus = None
site = None
owner = None
jobGroup = None
date = None
filename = "std.out"
if len(args) != 1:
Script.showHelp()
searchstring = str(args[0])
for switch in Script.getUnprocessedSwitches():
if switch[0].lower() == "status":
status = switch[1]
elif switch[0].lower() == "minorstatus":
minorStatus = switch[1]
elif switch[0].lower() == "applicationstatus":
appStatus = switch[1]
elif switch[0].lower() == "site":
site = switch[1]
elif switch[0].lower() == "owner":
owner = switch[1]
elif switch[0].lower() == "jobgroup":
jobGroup = switch[1]
elif switch[0].lower() == "date":
date = switch[1]
elif switch[0].lower() == "file":
filename = switch[1]
selDate = date
if not date:
selDate = "Today"
from DIRAC.Interfaces.API.Dirac import Dirac
dirac = Dirac()
exitCode = 0
errorList = []
resultDict = {}
result = dirac.selectJobs(
status=status,
minorStatus=minorStatus,
applicationStatus=appStatus,
site=site,
owner=owner,
jobGroup=jobGroup,
date=date,
)
if result["OK"]:
jobs = result["Value"]
else:
print("Error in selectJob", result["Message"])
DIRAC.exit(2)
for job in jobs:
result = dirac.getOutputSandbox(job)
if result["OK"]:
if os.path.exists("%s" % job):
lines = []
try:
lines = open(os.path.join(job, filename)).readlines()
except Exception as x:
errorList.append((job, x))
for line in lines:
if line.count(searchstring):
resultDict[job] = line
rmtree("%s" % (job))
else:
errorList.append((job, result["Message"]))
exitCode = 2
for result in resultDict.items():
print(result)
DIRAC.exit(exitCode)
Script.registerSwitch("S:", "setup=",
"set the software dist module to update.")
Script.registerSwitch("D:", "softwareDistModule=",
"set the software dist module to update.")
Script.parseCommandLine()
args = Script.getPositionalArgs()
from DIRAC import gConfig
cFile = ""
sMod = ""
vo = ""
setup = ""
for unprocSw in Script.getUnprocessedSwitches():
if unprocSw[0] in ("F", "file"):
cFile = unprocSw[1]
if unprocSw[0] in ("V", "vo"):
vo = unprocSw[1]
if unprocSw[0] in ("D", "softwareDistModule"):
sMod = unprocSw[1]
if unprocSw[0] in ("S", "setup"):
setup = unprocSw[1]
localCfg = CFG()
if cFile:
localConfigFile = cFile
else:
print("WORKSPACE: %s" % os.path.expandvars("$WORKSPACE"))
if os.path.isfile(
def main():
catalog = None
Script.registerSwitch("C:", "Catalog=", "Catalog to use")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(" requestName: a request name")
Script.registerArgument(" LFNs: single LFN or file with LFNs")
Script.registerArgument(["targetSE: target SE"])
Script.parseCommandLine()
for switch in Script.getUnprocessedSwitches():
if switch[0] == "C" or switch[0].lower() == "catalog":
catalog = switch[1]
args = Script.getPositionalArgs()
requestName = None
targetSEs = None
if len(args) < 3:
Script.showHelp(exitCode=1)
requestName = args[0]
lfnList = getLFNList(args[1])
targetSEs = list(
set([se for targetSE in args[2:] for se in targetSE.split(",")]))
gLogger.info("Will create request '%s' with 'ReplicateAndRegister' "
"operation using %s lfns and %s target SEs" %
(requestName, len(lfnList), len(targetSEs)))
from DIRAC.RequestManagementSystem.Client.Request import Request
from DIRAC.RequestManagementSystem.Client.Operation import Operation
from DIRAC.RequestManagementSystem.Client.File import File
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.Core.Utilities.List import breakListIntoChunks
lfnChunks = breakListIntoChunks(lfnList, 100)
multiRequests = len(lfnChunks) > 1
error = 0
count = 0
reqClient = ReqClient()
fc = FileCatalog()
requestIDs = []
for lfnChunk in lfnChunks:
metaDatas = fc.getFileMetadata(lfnChunk)
if not metaDatas["OK"]:
gLogger.error("unable to read metadata for lfns: %s" %
metaDatas["Message"])
error = -1
continue
metaDatas = metaDatas["Value"]
for failedLFN, reason in metaDatas["Failed"].items():
gLogger.error("skipping %s: %s" % (failedLFN, reason))
lfnChunk = set(metaDatas["Successful"])
if not lfnChunk:
gLogger.error("LFN list is empty!!!")
error = -1
continue
if len(lfnChunk) > Operation.MAX_FILES:
gLogger.error(
"too many LFNs, max number of files per operation is %s" %
Operation.MAX_FILES)
error = -1
continue
count += 1
request = Request()
request.RequestName = requestName if not multiRequests else "%s_%d" % (
requestName, count)
replicateAndRegister = Operation()
replicateAndRegister.Type = "ReplicateAndRegister"
replicateAndRegister.TargetSE = ",".join(targetSEs)
if catalog is not None:
replicateAndRegister.Catalog = catalog
for lfn in lfnChunk:
metaDict = metaDatas["Successful"][lfn]
opFile = File()
opFile.LFN = lfn
opFile.Size = metaDict["Size"]
if "Checksum" in metaDict:
# # should check checksum type, now assuming Adler32 (metaDict["ChecksumType"] = 'AD'
opFile.Checksum = metaDict["Checksum"]
opFile.ChecksumType = "ADLER32"
replicateAndRegister.addFile(opFile)
request.addOperation(replicateAndRegister)
putRequest = reqClient.putRequest(request)
if not putRequest["OK"]:
gLogger.error("unable to put request '%s': %s" %
(request.RequestName, putRequest["Message"]))
error = -1
continue
requestIDs.append(str(putRequest["Value"]))
if not multiRequests:
gLogger.always(
"Request '%s' has been put to ReqDB for execution." %
request.RequestName)
if multiRequests:
gLogger.always(
"%d requests have been put to ReqDB for execution, with name %s_<num>"
% (count, requestName))
if requestIDs:
gLogger.always("RequestID(s): %s" % " ".join(requestIDs))
gLogger.always(
"You can monitor requests' status using command: 'dirac-rms-request <requestName/ID>'"
)
DIRAC.exit(error)
def main():
"""
Main executive code
"""
Script.registerSwitch("", "Job=", " JobID[,jobID2,...]")
Script.registerSwitch("", "Transformation=", " transformation ID")
Script.registerSwitch("", "Tasks=", " Associated to --Transformation, list of taskIDs")
Script.registerSwitch("", "Verbose", " Print more information")
Script.registerSwitch("", "Terse", " Only print request status")
Script.registerSwitch("", "Full", " Print full request content")
Script.registerSwitch("", "Status=", " Select all requests in a given status")
Script.registerSwitch(
"", "Since=", " Associated to --Status, start date yyyy-mm-dd or nb of days (default= -one day"
)
Script.registerSwitch("", "Until=", " Associated to --Status, end date (default= now")
Script.registerSwitch("", "Maximum=", " Associated to --Status, max number of requests ")
Script.registerSwitch("", "Reset", " Reset Failed files to Waiting if any")
Script.registerSwitch("", "Force", " Force reset even if not Failed")
Script.registerSwitch(
"", "All", " (if --Status Failed) all requests, otherwise exclude irrecoverable failures"
)
Script.registerSwitch("", "FixJob", " Set job Done if the request is Done")
Script.registerSwitch("", "Cancel", " Cancel the request")
Script.registerSwitch("", "ListJobs", " List the corresponding jobs")
Script.registerSwitch("", "TargetSE=", " Select request only if that SE is in the targetSEs")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
(
"file: a file containing a list of requests (Comma-separated on each line)",
"request: a request ID or a unique request name",
),
mandatory=False,
)
Script.registerArgument(["request: a request ID or a unique request name"], mandatory=False)
Script.parseCommandLine()
import DIRAC
from DIRAC import gLogger
jobs = []
requestID = 0
transID = None
taskIDs = None
tasks = None
requests = []
full = False
verbose = False
status = None
until = None
since = None
terse = False
allR = False
reset = False
fixJob = False
maxRequests = 999999999999
cancel = False
listJobs = False
force = False
targetSE = set()
for switch in Script.getUnprocessedSwitches():
if switch[0] == "Job":
jobs = []
job = "Unknown"
try:
for arg in switch[1].split(","):
if os.path.exists(arg):
with open(arg, "r") as fp:
lines = fp.readlines()
for line in lines:
for job in line.split(","):
jobs += [int(job.strip())]
gLogger.notice("Found %d jobs in file %s" % (len(jobs), arg))
else:
jobs.append(int(arg))
except TypeError:
gLogger.fatal("Invalid jobID", job)
elif switch[0] == "Transformation":
try:
transID = int(switch[1])
except Exception:
gLogger.fatal("Invalid transID", switch[1])
elif switch[0] == "Tasks":
try:
taskIDs = [int(task) for task in switch[1].split(",")]
except Exception:
gLogger.fatal("Invalid tasks", switch[1])
elif switch[0] == "Full":
full = True
elif switch[0] == "Verbose":
verbose = True
elif switch[0] == "Terse":
terse = True
elif switch[0] == "All":
allR = True
elif switch[0] == "Reset":
reset = True
elif switch[0] == "Force":
force = True
elif switch[0] == "Status":
status = switch[1].capitalize()
elif switch[0] == "Since":
since = convertDate(switch[1])
elif switch[0] == "Until":
until = convertDate(switch[1])
elif switch[0] == "FixJob":
fixJob = True
elif switch[0] == "Cancel":
cancel = True
elif switch[0] == "ListJobs":
listJobs = True
elif switch[0] == "Maximum":
try:
maxRequests = int(switch[1])
except Exception:
pass
elif switch[0] == "TargetSE":
targetSE = set(switch[1].split(","))
if reset and not force:
status = "Failed"
if fixJob:
status = "Done"
if terse:
verbose = True
if status:
if not until:
until = datetime.datetime.utcnow()
if not since:
since = until - datetime.timedelta(hours=24)
from DIRAC.RequestManagementSystem.Client.ReqClient import ReqClient
from DIRAC.RequestManagementSystem.Client.ReqClient import printRequest, recoverableRequest
reqClient = ReqClient()
if transID:
if not taskIDs:
gLogger.fatal("If Transformation is set, a list of Tasks should also be set")
Script.showHelp(exitCode=2)
# In principle, the task name is unique, so the request name should be unique as well
# If ever this would not work anymore, we would need to use the transformationClient
# to fetch the ExternalID
requests = ["%08d_%08d" % (transID, task) for task in taskIDs]
allR = True
elif not jobs:
requests = []
# Get full list of arguments, with and without comma
for arg in [x.strip() for ar in Script.getPositionalArgs() for x in ar.split(",")]:
if os.path.exists(arg):
lines = open(arg, "r").readlines()
requests += [reqID.strip() for line in lines for reqID in line.split(",")]
gLogger.notice("Found %d requests in file" % len(requests))
else:
requests.append(arg)
allR = True
else:
res = reqClient.getRequestIDsForJobs(jobs)
if not res["OK"]:
gLogger.fatal("Error getting request for jobs", res["Message"])
DIRAC.exit(2)
if res["Value"]["Failed"]:
gLogger.error("No request found for jobs %s" % ",".join(sorted(str(job) for job in res["Value"]["Failed"])))
requests = sorted(res["Value"]["Successful"].values())
if requests:
allR = True
else:
DIRAC.exit(0)
if status and not requests:
allR = allR or status != "Failed"
res = reqClient.getRequestIDsList([status], limit=maxRequests, since=since, until=until)
if not res["OK"]:
gLogger.error("Error getting requests:", res["Message"])
DIRAC.exit(2)
requests = [reqID for reqID, _st, updTime in res["Value"] if updTime > since and updTime <= until and reqID]
gLogger.notice("Obtained %d requests %s between %s and %s" % (len(requests), status, since, until))
if not requests:
gLogger.notice("No request selected....")
Script.showHelp(exitCode=2)
okRequests = []
jobIDList = []
for reqID in requests:
# We allow reqID to be the requestName if it is unique
try:
# PEP-515 allows for underscore in numerical literals
# So a request name 00123_00456
# is interpreted as a requestID 12300456
# Using an exception here for non-string is not an option
if isinstance(reqID, six.string_types) and not reqID.isdigit():
raise ValueError()
requestID = int(reqID)
except (ValueError, TypeError):
requestID = reqClient.getRequestIDForName(reqID)
if not requestID["OK"]:
gLogger.notice(requestID["Message"])
continue
requestID = requestID["Value"]
request = reqClient.peekRequest(requestID)
if not request["OK"]:
gLogger.error(request["Message"])
DIRAC.exit(-1)
request = request["Value"]
if not request:
gLogger.error("no such request %s" % requestID)
continue
# If no operation as the targetSE, skip
if targetSE:
found = False
for op in request:
if op.TargetSE and targetSE.intersection(op.TargetSE.split(",")):
found = True
break
if not found:
continue
# keep a list of jobIDs if requested
if request.JobID and listJobs:
jobIDList.append(request.JobID)
if status and request.Status != status:
gLogger.notice(
"Request %s is not in requested status %s%s" % (reqID, status, " (cannot be reset)" if reset else "")
)
continue
if fixJob and request.Status == "Done" and request.JobID:
# The request is for a job and is Done, verify that the job is in the proper status
result = reqClient.finalizeRequest(request.RequestID, request.JobID, useCertificates=False)
if not result["OK"]:
gLogger.error("Error finalizing job", result["Message"])
else:
gLogger.notice("Job %d updated to %s" % (request.JobID, result["Value"]))
continue
if cancel:
if request.Status not in ("Done", "Failed"):
ret = reqClient.cancelRequest(requestID)
if not ret["OK"]:
gLogger.error("Error canceling request %s" % reqID, ret["Message"])
else:
gLogger.notice("Request %s cancelled" % reqID)
else:
gLogger.notice("Request %s is in status %s, not cancelled" % (reqID, request.Status))
elif allR or recoverableRequest(request):
okRequests.append(str(requestID))
if reset:
gLogger.notice("============ Request %s =============" % requestID)
ret = reqClient.resetFailedRequest(requestID, allR=allR)
if not ret["OK"]:
gLogger.error("Error resetting request %s" % requestID, ret["Message"])
else:
if len(requests) > 1:
gLogger.notice("\n===================================")
dbStatus = reqClient.getRequestStatus(requestID).get("Value", "Unknown")
printRequest(request, status=dbStatus, full=full, verbose=verbose, terse=terse)
if listJobs:
gLogger.notice("List of %d jobs:\n" % len(jobIDList), ",".join(str(jobID) for jobID in jobIDList))
if status and okRequests:
from DIRAC.Core.Utilities.List import breakListIntoChunks
gLogger.notice("\nList of %d selected requests:" % len(okRequests))
for reqs in breakListIntoChunks(okRequests, 100):
gLogger.notice(",".join(reqs))
from DIRAC import gLogger
from DIRAC.Interfaces.API.Job import Job
from DIRAC.TransformationSystem.Client.Transformation import Transformation
# Needed to test transformations with Filters
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.DataManager import DataManager
# Parse the arguments
args = Script.getPositionalArgs()
if len(args) != 1:
Script.showHelp()
directory = args[0]
UseFilter = None
for switch, switchValue in Script.getUnprocessedSwitches():
if switch == "UseFilter":
UseFilter = True if switchValue.lower() == "true" else False
# Let's first create the prodJobuction
prodJobType = "Merge"
transName = "testProduction_" + str(int(time.time()))
desc = "just test"
prodJob = Job()
prodJob._addParameter(prodJob.workflow, "PRODUCTION_ID", "string", "00012345",
"ProductionID")
prodJob._addParameter(prodJob.workflow, "JOB_ID", "string", "00006789",
"ProductionJobID")
prodJob._addParameter(prodJob.workflow, "eventType", "string", "TestEventType",
"Event Type of the prodJobuction")
def main():
# Register workflow parameter switch
Script.registerSwitch(
"p:", "parameter=",
"Parameters that are passed directly to the workflow")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"jobXMLfile: specify path to the Job XML file description")
Script.parseCommandLine()
# from DIRAC.Core.Workflow.Parameter import *
from DIRAC import gLogger
from DIRAC.Core.Workflow.Workflow import fromXMLFile
from DIRAC.Core.Utilities.Proxy import executeWithoutServerCertificate
from DIRAC.WorkloadManagementSystem.Client.JobReport import JobReport
from DIRAC.AccountingSystem.Client.DataStoreClient import DataStoreClient
from DIRAC.RequestManagementSystem.Client.Request import Request
# Forcing the current directory to be the first in the PYTHONPATH
sys.path.insert(0, os.path.realpath("."))
gLogger.showHeaders(True)
@executeWithoutServerCertificate
def jobexec(jobxml, wfParameters):
jobfile = os.path.abspath(jobxml)
if not os.path.exists(jobfile):
gLogger.warn("Path to specified workflow %s does not exist" %
(jobfile))
sys.exit(1)
workflow = fromXMLFile(jobfile)
gLogger.debug(workflow)
code = workflow.createCode()
gLogger.debug(code)
jobID = 0
if "JOBID" in os.environ:
jobID = os.environ["JOBID"]
gLogger.info("DIRAC JobID %s is running at site %s" %
(jobID, DIRAC.siteName()))
workflow.addTool("JobReport", JobReport(jobID))
workflow.addTool("AccountingReport", DataStoreClient())
workflow.addTool("Request", Request())
# Propagate the command line parameters to the workflow if any
for pName, pValue in wfParameters.items():
workflow.setValue(pName, pValue)
# Propagate the command line parameters to the workflow module instances of each step
for stepdefinition in workflow.step_definitions.values():
for moduleInstance in stepdefinition.module_instances:
for pName, pValue in wfParameters.items():
if moduleInstance.parameters.find(pName):
moduleInstance.parameters.setValue(pName, pValue)
return workflow.execute()
positionalArgs = Script.getPositionalArgs()
if len(positionalArgs) != 1:
gLogger.debug("Positional arguments were %s" % (positionalArgs))
DIRAC.abort(1, "Must specify the Job XML file description")
if "JOBID" in os.environ:
gLogger.info("JobID: %s" % (os.environ["JOBID"]))
jobXMLfile = positionalArgs[0]
parList = Script.getUnprocessedSwitches()
parDict = {}
for switch, parameter in parList:
if switch == "p":
name, value = parameter.split("=")
value = value.strip()
# The comma separated list in curly brackets is interpreted as a list
if value.startswith("{"):
value = value[1:-1].replace('"', "").replace(" ",
"").split(",")
value = ";".join(value)
parDict[name] = value
gLogger.debug("PYTHONPATH:\n%s" % ("\n".join(sys.path)))
jobExec = jobexec(jobXMLfile, parDict)
if not jobExec["OK"]:
gLogger.debug("Workflow execution finished with errors, exiting")
if jobExec["Errno"]:
os._exit(jobExec["Errno"])
else:
os._exit(1)
else:
gLogger.debug("Workflow execution successful, exiting")
# dirac_jobexec might interact with ARC library which cannot be closed using a simple sys.exit(0)
# See https://bugzilla.nordugrid.org/show_bug.cgi?id=4022 for further details
os._exit(0)
def main():
Script.registerSwitch("E:", "email=",
"Boolean True/False (True by default)")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument("Site: Name of the Site")
Script.registerArgument("Comment: Reason of the action")
Script.parseCommandLine(ignoreErrors=True)
from DIRAC.Interfaces.API.DiracAdmin import DiracAdmin
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC import exit as DIRACExit, gConfig, gLogger
def getBoolean(value):
if value.lower() == "true":
return True
elif value.lower() == "false":
return False
else:
Script.showHelp()
email = True
for switch in Script.getUnprocessedSwitches():
if switch[0] == "email":
email = getBoolean(switch[1])
diracAdmin = DiracAdmin()
exitCode = 0
errorList = []
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
print("ERROR: Could not contact Configuration Service")
exitCode = 2
DIRACExit(exitCode)
# result = promptUser(
# 'All the elements that are associated with this site will be active, '
# 'are you sure about this action?'
# )
# if not result['OK'] or result['Value'] is 'n':
# print 'Script stopped'
# DIRACExit( 0 )
# parseCommandLine show help when mandatory arguments are not specified or incorrect argument
site, comment = Script.getPositionalArgs(group=True)
result = diracAdmin.allowSite(site, comment, printOutput=True)
if not result["OK"]:
errorList.append((site, result["Message"]))
exitCode = 2
else:
if email:
userName = diracAdmin._getCurrentUser()
if not userName["OK"]:
print("ERROR: Could not obtain current username from proxy")
exitCode = 2
DIRACExit(exitCode)
userName = userName["Value"]
subject = "%s is added in site mask for %s setup" % (site, setup)
body = "Site %s is added to the site mask for %s setup by %s on %s.\n\n" % (
site,
setup,
userName,
time.asctime(),
)
body += "Comment:\n%s" % comment
addressPath = "EMail/Production"
address = Operations().getValue(addressPath, "")
if not address:
gLogger.notice(
"'%s' not defined in Operations, can not send Mail\n" %
addressPath, body)
else:
result = diracAdmin.sendMail(address, subject, body)
else:
print("Automatic email disabled by flag.")
for error in errorList:
print("ERROR %s: %s" % error)
DIRACExit(exitCode)
def main():
Script.registerSwitch("", "Path=", " Path to search for")
Script.registerSwitch(
"", "SE=",
" (comma-separated list of) SEs/SE-groups to be searched")
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
[
"metaspec: metadata index specification (of the form: "
'"meta=value" or "meta<value", "meta!=value", etc.)'
],
mandatory=False,
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
import DIRAC
from DIRAC import gLogger
from DIRAC.Resources.Catalog.FileCatalog import FileCatalog
from DIRAC.DataManagementSystem.Client.MetaQuery import MetaQuery, FILE_STANDARD_METAKEYS
from DIRAC.DataManagementSystem.Utilities.DMSHelpers import resolveSEGroup
path = "/"
seList = None
for opt, val in Script.getUnprocessedSwitches():
if opt == "Path":
path = val
elif opt == "SE":
seList = resolveSEGroup(val.split(","))
if seList:
args.append("SE=%s" % ",".join(seList))
fc = FileCatalog()
result = fc.getMetadataFields()
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
typeDict = result["Value"]["FileMetaFields"]
typeDict.update(result["Value"]["DirectoryMetaFields"])
# Special meta tags
typeDict.update(FILE_STANDARD_METAKEYS)
if len(args) < 1:
print("Error: No argument provided\n%s:" % Script.scriptName)
gLogger.notice("MetaDataDictionary: \n%s" % str(typeDict))
Script.showHelp(exitCode=1)
mq = MetaQuery(typeDict=typeDict)
result = mq.setMetaQuery(args)
if not result["OK"]:
gLogger.error("Illegal metaQuery:", result["Message"])
DIRAC.exit(-1)
metaDict = result["Value"]
path = metaDict.pop("Path", path)
result = fc.findFilesByMetadata(metaDict, path)
if not result["OK"]:
gLogger.error("Can not access File Catalog:", result["Message"])
DIRAC.exit(-1)
lfnList = sorted(result["Value"])
gLogger.notice("\n".join(lfn for lfn in lfnList))