def getSEsForSite( siteName, withSiteLocalSEMapping = False ):
""" Given a DIRAC site name this method returns a list of corresponding SEs.
"""
result = DMSHelpers().getSEsForSite( siteName, connectionLevel = 'DOWNLOAD' if withSiteLocalSEMapping else 'LOCAL' )
if not result['OK']:
return S_OK( [] )
return result
def export_getTree(self, elementType, elementName):
"""
Given an element type and name,
finds its parent site and returns all descendants of that site.
"""
gLogger.info('getTree')
site = self.getSite(elementType, elementName)
if not site:
return S_ERROR('No site')
siteStatus = rsClient.selectStatusElement(
'Site',
'Status',
name=site,
meta={'columns': ['StatusType', 'Status']})
if not siteStatus['OK']:
return siteStatus
tree = {site: {'statusTypes': dict(siteStatus['Value'])}}
ces = CSHelpers.getSiteComputingElements(site)
cesStatus = rsClient.selectStatusElement(
'Resource',
'Status',
name=ces,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not cesStatus['OK']:
return cesStatus
res = DMSHelpers().getSiteSEMapping()
if not res['OK']:
return res
ses = res['Value'][1].get(site, [])
sesStatus = rsClient.selectStatusElement(
'Resource',
'Status',
name=ses,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not sesStatus['OK']:
return sesStatus
def feedTree(elementsList):
elements = {}
for elementTuple in elementsList['Value']:
name, statusType, status = elementTuple
if name not in elements:
elements[name] = {}
elements[name][statusType] = status
return elements
tree[site]['ces'] = feedTree(cesStatus)
tree[site]['ses'] = feedTree(sesStatus)
return S_OK(tree)
def printSEInfo(voName):
fields = ('SE', 'Status', 'Protocols', 'Aliases')
records = []
for se in DMSHelpers(voName).getStorageElements(): # this will get the full list of SEs, not only the vo's ones.
seObject = StorageElement(se)
if not (seObject.vo and voName in seObject.vo.strip().split(',') or not seObject.voName):
continue
result = seObject.status()
status = []
for statusType in ['Write', 'Read']:
if result[statusType]:
status.append(statusType)
if status:
status = '/'.join(status)
else:
status = "InActive"
records.append((se, status,
",".join([seProtocol['Protocol'] for seProtocol in seObject.protocolOptions])))
gLogger.notice(printTable(fields, records, printOut=False, columnSeparator=' '))
return S_OK()
def isSameSiteSE( se1, se2 ): """ Check if the 2 SEs are at the same site """ dmsHelper = DMSHelpers() site1 = dmsHelper.getLocalSiteForSE( se1 ).get( 'Value' ) site2 = dmsHelper.getLocalSiteForSE( se2 ).get( 'Value' ) return site1 and site2 and site1 == site2
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param fromDict: if false, new request. Can be json string that represents the object, or the dictionary directly
"""
self.__waiting = None
now = datetime.datetime.utcnow().replace(microsecond=0)
self._CreationTime = now
self._SubmitTime = now
self._LastUpdate = now
# the time before which the request should not be executed
# If None, no delay
self._NotBefore = now
self._Status = "Done"
self.JobID = 0
self.Error = None
self.DIRACSetup = None
self.OwnerDN = None
self.RequestName = None
self.OwnerGroup = None
self._SourceComponent = None
self.dmsHelper = DMSHelpers()
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__operations__ = []
if isinstance(fromDict, six.string_types):
fromDict = json.loads(fromDict)
elif not isinstance(fromDict, dict):
fromDict = {}
if "Operations" in fromDict:
for opDict in fromDict.get("Operations", []):
self += Operation(opDict)
del fromDict["Operations"]
for key, value in fromDict.items():
# The JSON module forces the use of UTF-8, which is not properly
# taken into account in DIRAC.
# One would need to replace all the '== str' with 'in six.string_types'
# This is converting `unicode` to `str` and doesn't make sense in Python 3
if six.PY2 and isinstance(value, six.string_types):
value = value.encode()
if value:
setattr(self, key, value)
self._notify()
def __init__(self, vo=None):
"""The plugin is instanciated once per ``FTS3Operation``, so it is a
good place to do global initialization
:param str vo: Virtual Organization
"""
self.vo = vo
self.thirdPartyProtocols = DMSHelpers(vo=vo).getThirdPartyProtocols()
def getSitesForSE( storageElement, gridName = '', withSiteLocalSEMapping = False ):
""" Given a DIRAC SE name this method returns a list of corresponding sites.
Optionally restrict to Grid specified by name.
"""
result = DMSHelpers().getSitesForSE( storageElement, connectionLevel = 'DOWNLOAD' if withSiteLocalSEMapping else 'LOCAL' )
if not result['OK'] or not gridName:
return result
return S_OK( [site for site in result['Value'] if siteGridName( site ) == gridName] )
def __init__(self, fromDict=None):
"""c'tor
:param self: self reference
:param fromDict: if false, new request. Can be json string that represents the object, or the dictionary directly
"""
self.__waiting = None
now = datetime.datetime.utcnow().replace(microsecond=0)
self._CreationTime = now
self._SubmitTime = now
self._LastUpdate = now
# the time before which the request should not be executed
# If None, no delay
self._NotBefore = now
self._Status = "Done"
self.JobID = 0
self.Error = None
self.DIRACSetup = None
self.OwnerDN = None
self.RequestName = None
self.OwnerGroup = None
self._SourceComponent = None
self.dmsHelper = DMSHelpers()
proxyInfo = getProxyInfo()
if proxyInfo["OK"]:
proxyInfo = proxyInfo["Value"]
if proxyInfo["validGroup"] and proxyInfo["validDN"]:
self.OwnerDN = proxyInfo["identity"]
self.OwnerGroup = proxyInfo["group"]
self.__operations__ = []
if isinstance(fromDict, str):
fromDict = json.loads(fromDict)
elif not isinstance(fromDict, dict):
fromDict = {}
if "Operations" in fromDict:
for opDict in fromDict.get("Operations", []):
self += Operation(opDict)
del fromDict["Operations"]
for key, value in fromDict.items():
if value:
setattr(self, key, value)
self._notify()
def export_getTree(self, elementType, elementName):
"""
Given an element type and name,
finds its parent site and returns all descendants of that site.
"""
site = self.getSite(elementType, elementName)
if not site:
return S_ERROR('No site')
siteStatus = rsClient.selectStatusElement('Site', 'Status', name=site,
meta={'columns': ['StatusType', 'Status']})
if not siteStatus['OK']:
return siteStatus
tree = {site: {'statusTypes': dict(siteStatus['Value'])}}
result = getSiteCEMapping()
if not result['OK']:
return result
ces = result['Value'][site]
cesStatus = rsClient.selectStatusElement('Resource', 'Status', name=ces,
meta={'columns': ['Name', 'StatusType', 'Status']})
if not cesStatus['OK']:
return cesStatus
res = DMSHelpers().getSiteSEMapping()
if not res['OK']:
self.log.error('Could not get site to SE mapping', res['Message'])
return S_OK()
ses = res['Value'][1].get(site, [])
sesStatus = rsClient.selectStatusElement('Resource', 'Status', name=list(ses),
meta={'columns': ['Name', 'StatusType', 'Status']})
if not sesStatus['OK']:
return sesStatus
def feedTree(elementsList):
elements = {}
for elementTuple in elementsList['Value']:
name, statusType, status = elementTuple
if name not in elements:
elements[name] = {}
elements[name][statusType] = status
return elements
tree[site]['ces'] = feedTree(cesStatus)
tree[site]['ses'] = feedTree(sesStatus)
return S_OK(tree)
def _storeCommand(self, results):
"""
Stores the results in the cache (SpaceTokenOccupancyCache),
and adds records to the StorageOccupancy accounting.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0,
'SpaceReservation': 'LHCb-Disk'}
:returns: S_OK/S_ERROR dict
"""
# Stores in cache
res = self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results['Endpoint'],
lastCheckTime=datetime.utcnow(),
free=results['Free'],
total=results['Total'],
token=results['ElementName'])
if not res['OK']:
self.log.error("Error calling addOrModifySpaceTokenOccupancyCache",
res['Message'])
return res
# Now proceed with the accounting
siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName'])
if not siteRes['OK']:
return siteRes
accountingDict = {
'StorageElement': results['ElementName'],
'Endpoint': results['Endpoint'],
'Site': siteRes['Value'] if siteRes['Value'] else 'unassigned'
}
results['Used'] = results['Total'] - results['Free']
for sType in ['Total', 'Free', 'Used']:
spaceTokenAccounting = StorageOccupancy()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey('SpaceType', sType)
spaceTokenAccounting.setValueByKey(
'Space', int(convertSizeUnits(results[sType], 'MB', 'B')))
res = gDataStoreClient.addRegister(spaceTokenAccounting)
if not res['OK']:
self.log.warn("Could not commit register", res['Message'])
continue
return gDataStoreClient.commit()
def _storeCommand(self, results):
"""
Stores the results in the cache (SpaceTokenOccupancyCache),
and adds records to the StorageOccupancy accounting.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0}
:returns: S_OK/S_ERROR dict
"""
# Stores in cache
res = self.rmClient.addOrModifySpaceTokenOccupancyCache(
endpoint=results["Endpoint"],
lastCheckTime=datetime.utcnow(),
free=results["Free"],
total=results["Total"],
token=results["ElementName"],
)
if not res["OK"]:
self.log.error("Error calling addOrModifySpaceTokenOccupancyCache",
res["Message"])
return res
# Now proceed with the accounting
siteRes = DMSHelpers().getLocalSiteForSE(results["ElementName"])
if not siteRes["OK"]:
return siteRes
accountingDict = {
"StorageElement": results["ElementName"],
"Endpoint": results["Endpoint"],
"Site": siteRes["Value"] if siteRes["Value"] else "unassigned",
}
results["Used"] = results["Total"] - results["Free"]
for sType in ["Total", "Free", "Used"]:
spaceTokenAccounting = StorageOccupancy()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey("SpaceType", sType)
spaceTokenAccounting.setValueByKey(
"Space", int(convertSizeUnits(results[sType], "MB", "B")))
res = gDataStoreClient.addRegister(spaceTokenAccounting)
if not res["OK"]:
self.log.warn("Could not commit register", res["Message"])
continue
return gDataStoreClient.commit()
def __getSEsFromOptions(dmScript):
seList = dmScript.getOption('SEs', [])
sites = dmScript.getOption('Sites', [])
if sites:
siteSEs = []
dmsHelper = DMSHelpers()
for site in sites:
siteSEs += dmsHelper.getSEsForSite(site).get('Value', [])
if seList and siteSEs:
seList = list(set(seList) & set(siteSEs))
else:
seList += siteSEs
return seList
def __init__(self, argumentsDict):
"""Standard constructor"""
self.name = COMPONENT_NAME
self.log = gLogger.getSubLogger(self.name)
self.inputData = argumentsDict["InputData"]
self.configuration = argumentsDict["Configuration"]
# Warning: this contains not only the SEs but also the file metadata
self.fileCatalogResult = argumentsDict["FileCatalog"]
# By default put each input data file into a separate directory
self.inputDataDirectory = argumentsDict.get("InputDataDirectory", "PerFile")
self.jobID = None
self.counter = 1
self.availableSEs = DMSHelpers().getStorageElements()
def getSESiteMapping(gridName='', withSiteLocalSEMapping=False):
""" Returns a dictionary of all SEs and their associated site(s), e.g.
{'CERN-RAW':'LCG.CERN.ch','CERN-RDST':'LCG.CERN.ch',...]}
Although normally one site exists for a given SE, it is possible over all
Grid types to have multiple entries.
If gridName is specified, result is restricted to that Grid type.
Assumes CS structure of: /Resources/Sites/<GRIDNAME>/<SITENAME>
"""
storageElements = DMSHelpers().getStorageElements()
return S_OK(dict((se,
getSitesForSE(se, gridName=gridName,
withSiteLocalSEMapping=withSiteLocalSEMapping).get('Value', []))
for se in storageElements))
def doMaster(self):
"""
This method calls the doNew method for each storage element
that exists in the CS.
"""
for name in DMSHelpers().getStorageElements():
# keeping TB as default
diskSpace = self.doNew((name, 'MB'))
if not diskSpace['OK']:
gLogger.warn("Unable to calculate free/total disk space", "name: %s" % name)
gLogger.warn(diskSpace['Message'])
continue
return S_OK()
def __init__(self):
"""constructor"""
cmd.Cmd.__init__(self)
self.prompt = "$[/]$"
self.bk = LHCB_BKKDBClient()
self.diracAPI = DiracLHCb()
self.currentPath = '/'
self.do_setDataQualityFlags('OK')
self.saveParser = argparse.ArgumentParser(description="Save LFNS", prog='save')
self.saveParser.add_argument('filename', type=str, help='file name')
self.saveParser.add_argument("-f", "--format", help="txt or py")
self.saveParser.add_argument("-n", "--num", help="number of files to be saved")
self.saveParser.add_argument("-c", "--with-fileCatalog", help="save POOL XML catalog in a given site")
self.sites = {}
self.sites = DMSHelpers().getShortSiteNames(withStorage=False, tier=(0, 1))
def getSiteSEMapping( gridName = '', withSiteLocalSEMapping = False ):
""" Returns a dictionary of all sites and their localSEs as a list, e.g.
{'LCG.CERN.ch':['CERN-RAW','CERN-RDST',...]}
If gridName is specified, result is restricted to that Grid type.
"""
result = DMSHelpers().getSiteSEMapping()
if not result['OK']:
return result
if withSiteLocalSEMapping:
mapping = result['Value'][2]
else:
mapping = result['Value'][1]
if gridName:
mapping = dict( ( site, mapping[site] ) for site in mapping if siteGridName( site ) == gridName )
return S_OK( mapping )
def __init__(self, requestObject=None, log=None, defaultChecksumType='ADLER32'):
""" Constructor function, can specify request object to instantiate
FailoverTransfer or a new request object is created.
"""
self.log = log
if not self.log:
self.log = gLogger.getSubLogger("FailoverTransfer")
self.request = requestObject
if not self.request:
self.request = Request()
self.request.RequestName = 'noname_request'
self.request.SourceComponent = 'FailoverTransfer'
self.defaultChecksumType = defaultChecksumType
self.registrationProtocols = DMSHelpers().getRegistrationProtocols()
def getSiteElements(siteName):
"""
Gets all the computing and storage elements for a given site
"""
res = DMSHelpers().getSiteSEMapping()
if not res["OK"]:
return res
resources = res["Value"][1].get(siteName, [])
res = getQueues(siteName)
if not res["OK"]:
return res
resources = list(resources) + list(res["Value"].get(siteName, []))
return S_OK(resources)
def export_getSitesResources(self, siteNames):
"""
Returns dictionary with SEs and CEs for the given site(s). If siteNames is
None, all sites are taken into account.
:return: S_OK( { site1 : { ces : [ ces ], 'ses' : [ ses ] },... } ) | S_ERROR
"""
if siteNames is None:
res = getSites()
if not res["OK"]:
self.log.error("Error getting sites", res["Message"])
return res
siteNames = res["Value"]
if isinstance(siteNames, six.string_types):
siteNames = [siteNames]
sitesRes = {}
for siteName in siteNames:
result = getSiteCEMapping()
if not result["OK"]:
self.log.error("Error getting sites/CEs mapping",
result["Message"])
return result
res = {}
res["ces"] = result["Value"][siteName]
# Convert StorageElements to host names
result = DMSHelpers().getSiteSEMapping()
if not result["OK"]:
self.log.error("Error getting sites/SEs mapping",
result["Message"])
sitesRes[siteName] = res
continue
ses = result["Value"][1].get(siteName, [])
result = getStorageElementsHosts(ses)
if not result["OK"]:
self.log.error("Error getting storage element hosts",
result["Message"])
return result
# Remove duplicates
res["ses"] = list(set(result["Value"]))
sitesRes[siteName] = res
return S_OK(sitesRes)
def __init__(self,
plugin='Standard',
transClient=None,
dataManager=None,
fc=None,
debug=False,
transInThread=None,
transID=None):
"""
c'tor
Setting defaults
"""
# clients
if transClient is None:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if dataManager is None:
self.dm = DataManager()
else:
self.dm = dataManager
if fc is None:
self.fc = FileCatalog()
else:
self.fc = fc
self.dmsHelper = DMSHelpers()
self.plugin = plugin
self.transID = transID
self.params = {}
self.groupSize = 0
self.maxFiles = 0
self.cachedLFNSize = {}
self.transString = ''
self.debug = debug
if transInThread is None:
self.transInThread = {}
else:
self.transInThread = transInThread
self.log = gLogger.getSubLogger(self.plugin + self.transInThread.get(
self.transID, ' [NoThread] [%s] ' % self.transID))
# FIXME: This doesn't work (yet) but should soon, will allow scripts to get the context
self.log.showHeaders(True)
def _storeCommand(self, results):
""" _storeCommand
Adding records to accounting, on top of what does the derived method.
:param dict results: something like {'ElementName': 'CERN-HIST-EOS',
'Endpoint': 'httpg://srm-eoslhcb-bis.cern.ch:8443/srm/v2/server',
'Free': 3264963586.10073,
'Total': 8000000000.0,
'SpaceReservation': 'LHCb-Disk'}
:returns: S_OK/S_ERROR dict
"""
res = super(FreeDiskSpaceCommand, self)._storeCommand(results)
if not res['OK']:
return res
siteRes = DMSHelpers().getLocalSiteForSE(results['ElementName'])
if not siteRes['OK']:
return siteRes
if not siteRes['Value']:
return S_OK()
spaceReservation = results.get('SpaceReservation')
accountingDict = {
'SpaceToken': spaceReservation,
'Endpoint': results['Endpoint'],
'Site': siteRes['Value']
}
results['Used'] = results['Total'] - results['Free']
for sType in ['Total', 'Free', 'Used']:
spaceTokenAccounting = SpaceToken()
spaceTokenAccounting.setNowAsStartAndEndTime()
spaceTokenAccounting.setValuesFromDict(accountingDict)
spaceTokenAccounting.setValueByKey('SpaceType', sType)
spaceTokenAccounting.setValueByKey(
'Space', int(convertSizeUnits(results[sType], 'MB', 'B')))
gDataStoreClient.addRegister(spaceTokenAccounting)
gDataStoreClient.commit()
return S_OK()
def getStorageElementEndpoints(storageElements=None):
""" get the endpoints of the Storage ELements
"""
if storageElements is None:
storageElements = DMSHelpers().getStorageElements()
storageElementEndpoints = []
for se in storageElements:
seEndpoint = getStorageElementEndpoint(se)
if not seEndpoint['OK']:
continue
storageElementEndpoints.append(seEndpoint['Value'])
return S_OK(list(set(storageElementEndpoints)))
def __getSitesForSE(self, seName):
""" Returns a list of sites having the given SE as a local one.
Uses the local cache of the site-se information
"""
# Empty the cache if too old
now = time.time()
if (now - self.__lastCacheUpdate) > self.__cacheLifeTime:
self.log.verbose('Resetting the SE to site mapping cache')
self.__SEToSiteMap = {}
self.__lastCacheUpdate = now
if seName not in self.__SEToSiteMap:
result = DMSHelpers().getSitesForSE(seName)
if not result['OK']:
return result
self.__SEToSiteMap[seName] = list(result['Value'])
return S_OK(self.__SEToSiteMap[seName])
def getDIRACSesForHostName(hostName):
""" returns the DIRAC SEs that share the same hostName
:param str hostName: host name, e.g. 'storm-fe-lhcb.cr.cnaf.infn.it'
:return: S_OK with list of DIRAC SE names, or S_ERROR
"""
seNames = DMSHelpers().getStorageElements()
resultDIRACSEs = []
for seName in seNames:
res = getSEHosts(seName)
if not res['OK']:
return res
if hostName in res['Value']:
resultDIRACSEs.extend(seName)
return S_OK(resultDIRACSEs)
def __init__(self,
plugin='Standard',
transClient=None,
dataManager=None,
fc=None,
debug=False,
transInThread=None,
transID=None):
"""
c'tor
Setting defaults
"""
# clients
if transClient is None:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if dataManager is None:
self.dm = DataManager()
else:
self.dm = dataManager
if fc is None:
self.fc = FileCatalog()
else:
self.fc = fc
self.dmsHelper = DMSHelpers()
self.plugin = plugin
self.transID = transID
self.params = {}
self.groupSize = 0
self.maxFiles = 0
self.cachedLFNSize = {}
self.transString = ''
self.debug = debug
self.seConfig = {}
if transInThread is None:
self.transInThread = {}
else:
self.transInThread = transInThread
self.log = gLogger.getSubLogger("%s/PluginUtilities" % plugin)
def export_getFreeDiskSpace(self, site, token):
""" Exporting to web the
"""
endpoint2Site = {}
for seName in DMSHelpers().getStorageElements():
res = CSHelpers.getStorageElementEndpoint(seName)
if not res['OK']:
continue
if not res['Value'] in endpoint2Site:
endpoint2Site[res['Value']] = seName.split('-', 1)[0]
endpointSet = set()
if site:
if isinstance(site, basestring):
site = [site]
for ep, siteName in endpoint2Site.items():
if siteName in site:
endpointSet.add(ep)
if endpointSet:
endpoint = list(endpointSet)
else:
endpoint = None
res = rmClient.selectSpaceTokenOccupancyCache(endpoint=endpoint,
token=token)
if not res['OK']:
return res
spList = [dict(zip(res['Columns'], sp)) for sp in res['Value']]
for spd in spList:
try:
spd['Site'] = endpoint2Site[spd['Endpoint']]
except KeyError:
spd['Site'] = 'Unknown'
return S_OK(spList)
def getStorageElementsHosts(seNames=None):
""" Get the hosts of the Storage Elements
"""
seHosts = []
if seNames is None:
seNames = DMSHelpers().getStorageElements()
for seName in seNames:
seHost = getSEHost(seName)
if not seHost['OK']:
gLogger.warn("Could not get SE Host", "SE: %s" % seName)
continue
if seHost['Value']:
seHosts.append(seHost['Value'])
return S_OK(list(set(seHosts)))
def getResources():
"""
Gets all resources
"""
resources = DMSHelpers().getStorageElements()
fts = getFTS()
if fts["OK"]:
resources = resources + fts["Value"]
fc = getFileCatalogs()
if fc["OK"]:
resources = resources + fc["Value"]
res = getCESiteMapping()
if res["OK"]:
resources = resources + list(res["Value"])
return S_OK(resources)
def getResources():
"""
Gets all resources
"""
resources = DMSHelpers().getStorageElements()
fts = getFTS()
if fts['OK']:
resources = resources + fts['Value']
fc = getFileCatalogs()
if fc['OK']:
resources = resources + fc['Value']
ce = getComputingElements()
if ce['OK']:
resources = resources + ce['Value']
return S_OK(resources)
