def getDirectoryMetadata(self, path):
"""Get metadata associated to the directory"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.getFileMetadata: Attempting to obtain metadata for %s directories." % len(urls))
serviceClient = Client(url=self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res["OK"]:
if res["Value"]["Exists"]:
if res["Value"]["Type"] == "Directory":
res["Value"]["Directory"] = True
gLogger.debug("DIPStorage.getFileMetadata: Successfully obtained metadata for %s." % url)
successful[url] = res["Value"]
else:
failed[url] = "Supplied path is not a directory"
else:
failed[url] = "Directory does not exist"
else:
gLogger.error(
"DIPStorage.getFileMetadata: Failed to get metadata for url", "%s: %s" % (url, res["Message"])
)
failed[url] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def isDirectory(self, path):
"""Determine whether the path is a directory"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.isDirectory: Attempting to determine whether %s paths are directories." % len(urls))
serviceClient = Client(url=self.url)
for url in urls:
res = serviceClient.getMetadata(url)
if res["OK"]:
if res["Value"]["Exists"]:
if res["Value"]["Type"] == "Directory":
gLogger.debug("DIPStorage.isDirectory: Successfully obtained metadata for %s." % url)
successful[url] = True
else:
successful[url] = False
else:
failed[url] = "Path does not exist"
else:
gLogger.error(
"DIPStorage.isDirectory: Failed to get metadata for url", "%s: %s" % (url, res["Message"])
)
failed[url] = res["Message"]
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
opsH = Operations()
self.maxResetCounter = opsH.getValue( 'Productions/ProductionFilesMaxResetCounter', 10 )
self.setServer( 'Transformation/TransformationManager' )
def uploadProxy(self,
proxy=None,
restrictLifeTime: int = 0,
rfcIfPossible=None):
"""Upload a proxy to the proxy management service using delegation
:param X509Chain proxy: proxy as a chain
:param restrictLifeTime: proxy live time in a seconds
:return: S_OK(dict)/S_ERROR() -- dict contain proxies
"""
if rfcIfPossible is not None:
if os.environ.get("DIRAC_DEPRECATED_FAIL", None):
raise NotImplementedError(
"'rfcIfPossible' argument is deprecated.")
gLogger.warn("'rfcIfPossible' argument is deprecated.")
# Discover proxy location
if isinstance(proxy, X509Chain):
chain = proxy
proxyLocation = ""
else:
if not proxy:
proxyLocation = Locations.getProxyLocation()
if not proxyLocation:
return S_ERROR("Can't find a valid proxy")
elif isinstance(proxy, str):
proxyLocation = proxy
else:
return S_ERROR("Can't find a valid proxy")
chain = X509Chain()
result = chain.loadProxyFromFile(proxyLocation)
if not result["OK"]:
return S_ERROR(
f"Can't load {proxyLocation}: {result['Message']}")
# Make sure it's valid
if chain.hasExpired().get("Value"):
return S_ERROR(f"Proxy {proxyLocation} has expired")
if chain.getDIRACGroup(ignoreDefault=True).get(
"Value") or chain.isVOMS().get("Value"):
return S_ERROR(
"Cannot upload proxy with DIRAC group or VOMS extensions")
rpcClient = Client(url="Framework/ProxyManager", timeout=120)
# Get a delegation request
result = rpcClient.requestDelegationUpload()
if not result["OK"]:
return result
reqDict = result["Value"]
# Generate delegated chain
chainLifeTime = chain.getRemainingSecs()["Value"] - 60
if restrictLifeTime and restrictLifeTime < chainLifeTime:
chainLifeTime = restrictLifeTime
result = chain.generateChainFromRequestString(reqDict["request"],
lifetime=chainLifeTime)
if result["OK"]:
result = rpcClient.completeDelegationUpload(
reqDict["id"], pemChain := result["Value"])
return result
def __init__(self, **kwargs):
""" Simple constructor
"""
Client.__init__(self, **kwargs)
self.setServer('WorkloadManagement/Matcher')
def __init__( self, **kwargs ):
"""
Constructor function
"""
Client.__init__( self, **kwargs )
self.setServer( 'Framework/ComponentMonitoring' )
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
## setup logger
self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL( "RequestManagement/localURL" )
if local:
self.__requestRPCClientsDict.setdefault( "local" , [ self.__requestRPCClient( local ) ] )
## central if any defined
central = PathFinder.getServiceURL( "RequestManagement/centralURL" )
if central:
self.__requestRPCClientsDict.setdefault( "central", [ self.__requestRPCClient( central ) ] )
## voboxes if any defined
voBoxUrls = fromChar( PathFinder.getServiceURL( "RequestManagement/voBoxURLs" ) )
if voBoxUrls:
self.__requestRPCClientsDict.setdefault( "voboxes", [] )
for voBoxURL in randomize( voBoxUrls ):
self.__requestRPCClientsDict["voboxes"].append( self.__requestRPCClient( voBoxURL ) )
self.setServer( 'RequestManagement/centralURL' )
def __init__(self, useCertificates=False):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__(self)
## setup logger
self.log = gLogger.getSubLogger("RequestManagement/RequestClient")
## dict to store all RPC clients for easy reuse
self.__requestRPCClientsDict = {}
## local if any defined
local = PathFinder.getServiceURL("RequestManagement/localURL")
if local:
self.__requestRPCClientsDict.setdefault(
"local", [self.__requestRPCClient(local)])
## central if any defined
central = PathFinder.getServiceURL("RequestManagement/centralURL")
if central:
self.__requestRPCClientsDict.setdefault(
"central", [self.__requestRPCClient(central)])
## voboxes if any defined
voBoxUrls = fromChar(
PathFinder.getServiceURL("RequestManagement/voBoxURLs"))
if voBoxUrls:
self.__requestRPCClientsDict.setdefault("voboxes", [])
for voBoxURL in randomize(voBoxUrls):
self.__requestRPCClientsDict["voboxes"].append(
self.__requestRPCClient(voBoxURL))
self.setServer('RequestManagement/centralURL')
def __init__(self, **kwargs):
"""
Constructor function
"""
Client.__init__(self, **kwargs)
self.setServer('Framework/ComponentMonitoring')
def listDirectory(self, path):
"""List the contents of the directory"""
res = checkArgumentFormat(path)
if not res["OK"]:
return res
urls = res["Value"]
successful = {}
failed = {}
gLogger.debug("DIPStorage.listDirectory: Attempting to list %s directories." % len(urls))
serviceClient = Client(url=self.url)
for url in urls:
res = serviceClient.listDirectory(url, "l")
if not res["OK"]:
failed[url] = res["Message"]
else:
files = {}
subDirs = {}
for subPath, pathDict in res["Value"].items():
if pathDict["Type"] == "File":
files[subPath] = pathDict
elif pathDict["Type"] == "Directory":
subDirs[subPath] = pathDict
successful[url] = {}
successful[url]["SubDirs"] = subDirs
successful[url]["Files"] = files
resDict = {"Failed": failed, "Successful": successful}
return S_OK(resDict)
def __init__( self, host, port = None, **kwargs ):
""" Constructor function. Takes a mandatory host parameter
"""
Client.__init__( self, **kwargs )
if not port:
port = SYSADMIN_PORT
self.setServer( 'dips://%s:%s/Framework/SystemAdministrator' % ( host, port ) )
def __init__(self, url=None, **kwargs):
""" Constructor function.
"""
Client.__init__(self, **kwargs)
self.setServer('DataManagement/FTS3Manager')
if url:
self.setServer(url)
def __init__(self, **kwargs):
""" Simple constructor
"""
Client.__init__(self, **kwargs)
self.setServer('WorkloadManagement/Matcher')
def __init__(self, url=None, **kwargs):
""" Constructor function.
"""
Client.__init__(self, **kwargs)
self.setServer('DataManagement/FTS3Manager')
if url:
self.setServer(url)
def setPersistency(self, userDN, userGroup, persistent):
"""Set the persistency for user/group
:param str userDN: user DN
:param str userGroup: user group
:param boolean persistent: presistent flag
:return: S_OK()/S_ERROR()
"""
# Hack to ensure bool in the rpc call
persistentFlag = True
if not persistent:
persistentFlag = False
rpcClient = Client(url="Framework/ProxyManager", timeout=120)
retVal = rpcClient.setPersistency(userDN, userGroup, persistentFlag)
if not retVal["OK"]:
return retVal
# Update internal persistency cache
cacheKey = (userDN, userGroup)
record = self.__usersCache.get(cacheKey, 0)
if record:
record["persistent"] = persistentFlag
self.__usersCache.add(
cacheKey,
self.__getSecondsLeftToExpiration(record["expirationtime"]),
record)
return retVal
class FileCatalogProxyClient(object):
"""File catalog client for the File Catalog proxy service"""
def __init__(self, fcName, **kwargs):
"""Constructor of the LCGFileCatalogProxy client class"""
self.method = None
self.fcName = fcName
self.rpc = Client(url="DataManagement/FileCatalogProxy", timeout=120)
self.valid = False
self.valid = self.rpc.ping()["OK"]
self.interfaceMethods = None
def isOK(self):
"""Is the Catalog available?"""
return self.valid
def getName(self):
"""Get the file catalog name"""
return self.fcName
def setInterfaceMethods(self, methodTuple):
self.interfaceMethods = methodTuple
def getInterfaceMethods(self):
return self.interfaceMethods
def __getattr__(self, name):
self.method = name
return self.execute
def execute(self, *parms, **kws):
"""Magic method dispatcher"""
return self.rpc.callProxyMethod(self.fcName, self.method, parms, kws)
def __init__(self, host, port=None, **kwargs):
""" Constructor function. Takes a mandatory host parameter
"""
Client.__init__(self, **kwargs)
if not port:
port = SYSADMIN_PORT
self.setServer('dips://%s:%s/Framework/SystemAdministrator' % (host, port))
def __init__( self, **kwargs ): """ :param self: self reference :param str url: service URL """ Client.__init__( self, **kwargs ) self.setServer( "DataManagement/DataLogging" ) self.setTimeout( 120 )
def __init__( self, url=None, **kwargs ):
""" Constructor function.
"""
Client.__init__( self, **kwargs )
self.setServer('DataManagement/FileCatalog')
if url:
self.setServer(url)
self.available = False
def __init__(self, **kwargs):
"""
:param self: self reference
:param str url: service URL
"""
Client.__init__(self, **kwargs)
self.setServer("DataManagement/DataLogging")
self.setTimeout(120)
def __init__(self, **kwargs):
""" Simple constructor
"""
Client.__init__(self, **kwargs)
self.setServer('Production/ProductionManager')
self.prodDescription = {}
self.stepCounter = 1
def __init__( self ):
"""c'tor
:param self: self reference
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "RequestManagement/ReqClient/pid_%s" % ( os.getpid() ) )
self.setServer( "RequestManagement/ReqManager" )
def __init__(self, fcName, **kwargs):
"""Constructor of the LCGFileCatalogProxy client class"""
self.method = None
self.fcName = fcName
self.rpc = Client(url="DataManagement/FileCatalogProxy", timeout=120)
self.valid = False
self.valid = self.rpc.ping()["OK"]
self.interfaceMethods = None
def __init__(self):
"""c'tor
:param self: self reference
"""
Client.__init__(self)
self.log = gLogger.getSubLogger("WorkloadManagement/VMClient")
self.setServer("WorkloadManagement/VirtualMachineManager")
def __init__(self, **kwargs):
Client.__init__(self, **kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue(
'Productions/ProductionFilesMaxResetCounter', 10)
self.setServer('Transformation/TransformationManager')
def __init__( self):
Client.__init__( self )
self.setServer( "DataManagement/Test" )
url = PathFinder.getServiceURL( "DataManagement/Test" )
if not url:
raise RuntimeError( "CS option DataManagement/Test URL is not set!" )
self.testManager = RPCClient( url )
def __init__(self, url=None, **kwargs):
""" Constructor function.
"""
Client.__init__(self, **kwargs)
self.setServer('DataManagement/FileCatalog')
if url:
self.setServer(url)
self.available = False
def uploadProxy(self, proxy=None, restrictLifeTime=0, rfcIfPossible=False):
"""Upload a proxy to the proxy management service using delegation
:param X509Chain proxy: proxy as a chain
:param int restrictLifeTime: proxy live time in a seconds
:param boolean rfcIfPossible: make rfc proxy if possible
:return: S_OK(dict)/S_ERROR() -- dict contain proxies
"""
# Discover proxy location
if isinstance(proxy, X509Chain):
chain = proxy
proxyLocation = ""
else:
if not proxy:
proxyLocation = Locations.getProxyLocation()
if not proxyLocation:
return S_ERROR("Can't find a valid proxy")
elif isinstance(proxy, six.string_types):
proxyLocation = proxy
else:
return S_ERROR("Can't find a valid proxy")
chain = X509Chain()
result = chain.loadProxyFromFile(proxyLocation)
if not result["OK"]:
return S_ERROR("Can't load %s: %s " %
(proxyLocation, result["Message"]))
# Make sure it's valid
if chain.hasExpired().get("Value"):
return S_ERROR("Proxy %s has expired" % proxyLocation)
if chain.getDIRACGroup(ignoreDefault=True).get(
"Value") or chain.isVOMS().get("Value"):
return S_ERROR(
"Cannot upload proxy with DIRAC group or VOMS extensions")
rpcClient = Client(url="Framework/ProxyManager", timeout=120)
# Get a delegation request
result = rpcClient.requestDelegationUpload(
chain.getRemainingSecs()["Value"])
if not result["OK"]:
return result
reqDict = result["Value"]
# Generate delegated chain
chainLifeTime = chain.getRemainingSecs()["Value"] - 60
if restrictLifeTime and restrictLifeTime < chainLifeTime:
chainLifeTime = restrictLifeTime
retVal = chain.generateChainFromRequestString(reqDict["request"],
lifetime=chainLifeTime,
rfc=rfcIfPossible)
if not retVal["OK"]:
return retVal
# Upload!
result = rpcClient.completeDelegationUpload(reqDict["id"],
retVal["Value"])
if not result["OK"]:
return result
return S_OK(result.get("proxies") or result["Value"])
def __init__( self, url = "DataManagement/DataLogging" ):
""" c'tor
:param self: self reference
:param str url: service URL
"""
Client.__init__( self )
self.setServer( url )
self.setTimeout( 120 )
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "DataManagement/FTSClient" )
self.setServer( "DataManagement/FTSManager" )
def getComponentsStatus(self, condDict):
"""This method is used to get the status of the components.
:type condDict: dictionary
:param condDict: A condition dictionary.
:return: S_OK with status and message about the component.
"""
rpcClient = Client(url="Framework/Monitoring", timeout=100)
return rpcClient.getComponentsStatus(condDict)
def __init__( self, **kwargs ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self, **kwargs )
self.log = gLogger.getSubLogger( "RequestManagement/RequestClient" )
self.setServer( "RequestManagement/RequestManager" )
def deleteProxyBundle(self, idList):
"""delete a list of id's
:param list,tuple idList: list of identity numbers
:return: S_OK(int)/S_ERROR()
"""
rpcClient = Client(url="Framework/ProxyManager", timeout=120)
return rpcClient.deleteProxyBundle(idList)
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "RequestManagement/ReqClient" )
self.setServer( "RequestManagement/ReqManager" )
def getUserProxiesInfo(self):
"""Get the user proxies uploaded info
:return: S_OK(dict)/S_ERROR()
"""
result = Client(url="Framework/ProxyManager",
timeout=120).getUserProxiesInfo()
if "rpcStub" in result:
result.pop("rpcStub")
return result
def __init__(self, url=None, **kwargs):
"""
c'tor
:param str url: can specify a specific URL
"""
Client.__init__(self, **kwargs)
self.setServer('Bookkeeping/BookkeepingManager')
if url:
self.setServer(url)
self.setTimeout(3600)
def __init__(self, **kwargs):
""" Simple constructor
"""
Client.__init__(self, **kwargs)
opsH = Operations()
self.maxResetCounter = opsH.getValue(
'Transformations/FilesMaxResetCounter', 10)
self.setServer('Transformation/TransformationManager')
def doCommand(self):
"""
The Command pings a service on a vobox, it needs a service URL to ping it.
:returns: a dict with the following:
.. code-block:: python
{
'serviceUpTime' : <serviceUpTime>,
'machineUpTime' : <machineUpTime>,
'site' : <site>,
'system' : <system>,
'service' : <service>
}
"""
# INPUT PARAMETERS
if "serviceURL" not in self.args:
return self.returnERROR(
S_ERROR('"serviceURL" not found in self.args'))
serviceURL = self.args["serviceURL"]
##
parsed = parse.urlparse(serviceURL)
site = parsed[1].split(":")[0]
try:
system, service = parsed[2].strip("/").split("/")
except ValueError:
return self.returnERROR(
S_ERROR('"%s" seems to be a malformed url' % serviceURL))
pinger = Client(url=serviceURL)
resPing = pinger.ping()
if not resPing["OK"]:
return self.returnERROR(resPing)
serviceUpTime = resPing["Value"].get("service uptime", 0)
machineUpTime = resPing["Value"].get("host uptime", 0)
result = {
"site": site,
"system": system,
"service": service,
"serviceUpTime": serviceUpTime,
"machineUpTime": machineUpTime,
}
return S_OK(result)
def deleteRemoteProxy(userdn, vogroup):
"""
Deletes proxy for a vogroup for the user envoking this function.
Returns a list of all deleted proxies (if any).
"""
rpcClient = Client(url="Framework/ProxyManager")
retVal = rpcClient.deleteProxyBundle([(userdn, vogroup)])
if retVal["OK"]:
gLogger.notice("Deleted proxy for %s." % vogroup)
else:
gLogger.error("Failed to delete proxy for %s." % vogroup)
def requestToken(self, requesterDN, requesterGroup, numUses=1):
"""Request a number of tokens. usesList must be a list of integers and each integer is the number of uses a token
must have
:param str requesterDN: user DN
:param str requesterGroup: user group
:param int numUses: number of uses
:return: S_OK(tuple)/S_ERROR() -- tuple contain token, number uses
"""
rpcClient = Client(url="Framework/ProxyManager", timeout=120)
return rpcClient.generateToken(requesterDN, requesterGroup, numUses)
def __init__( self, url = False, **kwargs ):
""" Constructor of the LCGFileCatalogProxy client class
"""
Client.__init__( self, **kwargs )
self.method = None
self.name = 'LFCProxy'
self.valid = False
self.setServer( 'DataManagement/LcgFileCatalogProxy' )
if url:
self.setServer( url )
self.setTimeout( 120 )
self.call = 'ping'
self.valid = self.executeRPC()['OK']
def __init__( self, useCertificates = False ):
"""c'tor
:param self: self reference
:param bool useCertificates: flag to enable/disable certificates
"""
Client.__init__( self )
self.log = gLogger.getSubLogger( "DataManagement/FTSClient" )
self.setServer( "DataManagement/FTSManager" )
# getting other clients
self.ftsValidator = FTSValidator()
self.dataManager = DataManager()
self.storageFactory = StorageFactory()
url = PathFinder.getServiceURL( "DataManagement/FTSManager" )
if not url:
raise RuntimeError( "CS option DataManagement/FTSManager URL is not set!" )
self.ftsManager = RPCClient( url )
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer( 'DataManagement/DataIntegrity' )
self.dm = DataManager()
self.fc = FileCatalog()
def __init__(self, **kwargs):
Client.__init__(self, **kwargs)
self.setServer('WorkloadManagement/PilotsLogging')
def __init__(self, **kwargs):
Client.__init__(self, **kwargs)
self.setServer("Overlay/Overlay")
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer( 'Transformation/TransformationManager' )
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer('DataManagement/FileCatalog')
def __init__(self, **kwargs ):
Client.__init__(self, **kwargs )
self.setServer("ResourceStatus/SoftwareTag")
def __init__(self, **kwargs ):
Client.__init__(self, **kwargs )
self.setServer('ProcessProduction/ProcessManager')
def setServer( self, url ): Client.setServer( self, url ) self.__requestManager = None self.requestManager()
def __init__( self, **kwargs ): Client.__init__( self, **kwargs ) self.setServer( 'StorageManagement/StorageManager' )
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer('WorkloadManagement/JobMonitoring')
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer( 'DataManagement/DataIntegrity' )
self.rm = ReplicaManager()
def __init__( self, **kwargs ):
Client.__init__( self, **kwargs )
self.setServer( 'WorkloadManagement/JobMonitoring' )
self.monitoringHandler = self._getRPC()
def __init__( self, **kwargs ): Client.__init__( self, **kwargs ) self.setServer( 'WorkloadManagement/PilotsLogging' ) self.pilotsLoggingHandler = self._getRPC()