def doNew(self, masterParams=None):
"""
Gets the total and the free disk space of a DIPS storage element that
is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
"""
if masterParams is not None:
elementName = masterParams
else:
elementName = self._prepareCommand()
if not elementName['OK']:
return elementName
se = StorageElement(elementName)
elementURL = se.getStorageParameters(protocol="dips")
if elementURL['OK']:
elementURL = se.getStorageParameters(
protocol="dips")['Value']['URLBase']
else:
gLogger.verbose("Not a DIPS storage element, skipping...")
return S_OK()
self.rpc = RPCClient(elementURL, timeout=120)
free = self.rpc.getFreeDiskSpace("/")
if not free['OK']:
return free
free = free['Value']
total = self.rpc.getTotalDiskSpace("/")
if not total['OK']:
return total
total = total['Value']
if free and free < 1:
free = 1
if total and total < 1:
total = 1
result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
endpoint=elementURL,
lastCheckTime=datetime.utcnow(),
free=free,
total=total,
token=elementName)
if not result['OK']:
return result
return S_OK()
def doNew(self, masterParams=None):
"""
Gets the total and the free disk space of a DIPS storage element that
is found in the CS and inserts the results in the SpaceTokenOccupancyCache table
of ResourceManagementDB database.
"""
if masterParams is not None:
elementName = masterParams
else:
elementName = self._prepareCommand()
if not elementName["OK"]:
return elementName
se = StorageElement(elementName)
elementURL = se.getStorageParameters(protocol="dips")
if elementURL["OK"]:
elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"]
else:
gLogger.verbose("Not a DIPS storage element, skipping...")
return S_OK()
self.rpc = RPCClient(elementURL, timeout=120)
free = self.rpc.getFreeDiskSpace("/")
if not free["OK"]:
return free
free = free["Value"]
total = self.rpc.getTotalDiskSpace("/")
if not total["OK"]:
return total
total = total["Value"]
if free and free < 1:
free = 1
if total and total < 1:
total = 1
result = self.rsClient.addOrModifySpaceTokenOccupancyCache(
endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName
)
if not result["OK"]:
return result
return S_OK()
def getSEParameters(seName):
"""get all the SE parameters in a list
:param str seName: name of the Storage Element
:return: S_OK() with list of dict with parameters
"""
# This import is here to avoid circular imports
from DIRAC.Resources.Storage.StorageElement import StorageElement
se = StorageElement(seName, hideExceptions=True)
protocolsSet = set(se.localAccessProtocolList) | set(
se.localWriteProtocolList)
seParametersList = []
for protocol in protocolsSet:
seParameters = se.getStorageParameters(protocol=protocol)
if seParameters["OK"]:
seParametersList.append(seParameters["Value"])
else:
gLogger.verbose("No SE parameters obtained",
"for SE %s and protocol %s" % (seName, protocol))
return S_OK(seParametersList)
def _getSEParameters(seName):
se = StorageElement(seName, hideExceptions=True)
pluginsList = se.getPlugins()
if not pluginsList['OK']:
gLogger.warn(pluginsList['Message'], "SE: %s" % seName)
return pluginsList
pluginsList = pluginsList['Value']
if 'SRM2' in pluginsList:
pluginsList.remove('SRM2')
pluginsList.insert(0, 'SRM2')
for plugin in pluginsList:
seParameters = se.getStorageParameters(plugin)
if seParameters['OK']:
break
return seParameters
def _getSEParameters( seName ):
se = StorageElement( seName, hideExceptions = True )
pluginsList = se.getPlugins()
if not pluginsList['OK']:
gLogger.warn( pluginsList['Message'], "SE: %s" % seName )
return pluginsList
pluginsList = pluginsList['Value']
if 'SRM2' in pluginsList:
pluginsList.remove( 'SRM2' )
pluginsList.insert( 0, 'SRM2' )
for plugin in pluginsList:
seParameters = se.getStorageParameters( plugin )
if seParameters['OK']:
break
return seParameters
def __fetchSpaceToken(seName):
""" Fetch the space token of storage element
:param seName name of the storageElement
:returns space token
"""
seToken = None
if seName:
seObj = StorageElement(seName)
res = seObj.getStorageParameters(protocol='srm')
if not res['OK']:
return res
seToken = res["Value"].get("SpaceToken")
return S_OK(seToken)
def _getSEParameters(seName):
se = StorageElement(seName, hideExceptions=True)
pluginsList = se.getPlugins()
if not pluginsList['OK']:
gLogger.warn(pluginsList['Message'], "SE: %s" % seName)
return pluginsList
pluginsList = pluginsList['Value']
# Put the srm capable protocol first, but why doing that is a
# mystery that will eventually need to be sorted out...
for plugin in ('GFAL2_SRM2', 'SRM2'):
if plugin in pluginsList:
pluginsList.remove(plugin)
pluginsList.insert(0, plugin)
for plugin in pluginsList:
seParameters = se.getStorageParameters(plugin)
if seParameters['OK']:
break
return seParameters
def _getSEParameters( seName ):
se = StorageElement( seName, hideExceptions = True )
pluginsList = se.getPlugins()
if not pluginsList['OK']:
gLogger.warn( pluginsList['Message'], "SE: %s" % seName )
return pluginsList
pluginsList = pluginsList['Value']
# Put the srm capable protocol first, but why doing that is a
# mystery that will eventually need to be sorted out...
for plugin in ( 'GFAL2_SRM2', 'SRM2' ):
if plugin in pluginsList:
pluginsList.remove( plugin )
pluginsList.insert( 0, plugin )
for plugin in pluginsList:
seParameters = se.getStorageParameters( plugin )
if seParameters['OK']:
break
return seParameters
def __fetchSpaceToken(seName):
""" Fetch the space token of storage element
:param seName name of the storageElement
:returns space token. If there is no SpaceToken defined, returns None
"""
seToken = None
if seName:
seObj = StorageElement(seName)
res = seObj.getStorageParameters(protocol='srm')
if not res['OK']:
# If there is no SRM protocol, we do not specify
# the space token
if cmpError(res, errno.ENOPROTOOPT):
return S_OK(None)
return res
seToken = res["Value"].get("SpaceToken")
return S_OK(seToken)
def __fetchSpaceToken(seName):
""" Fetch the space token of storage element
:param seName name of the storageElement
:returns space token. If there is no SpaceToken defined, returns None
"""
seToken = None
if seName:
seObj = StorageElement(seName)
res = seObj.getStorageParameters(protocol='srm')
if not res['OK']:
# If there is no SRM protocol, we do not specify
# the space token
if cmpError(res, errno.ENOPROTOOPT):
return S_OK(None)
return res
seToken = res["Value"].get("SpaceToken")
return S_OK(seToken)
def __submit( self, request, operation, toSubmit ):
""" create and submit new FTSJobs using list of FTSFiles
:param Request request: ReqDB.Request instance
:param list ftsFiles: list of FTSFile instances
:return: [ FTSJob, FTSJob, ...]
"""
log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) )
bySourceAndTarget = {}
for ftsFile in toSubmit:
if ftsFile.SourceSE not in bySourceAndTarget:
bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )
ftsJobs = []
for source, targetDict in bySourceAndTarget.iteritems():
for target, ftsFileList in targetDict.iteritems():
log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )
route = self.__ftsPlacement.findRoute( source, target )
if not route["OK"]:
log.error( route["Message"] )
continue
route = route["Value"]
routeValid = self.__ftsPlacement.isRouteValid( route )
if not routeValid['OK']:
log.error( "Route invalid : %s" % routeValid['Message'] )
continue
sourceSE = StorageElement( source )
sourceToken = sourceSE.getStorageParameters( "SRM2" )
if not sourceToken["OK"]:
log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) )
continue
seStatus = sourceSE.getStatus()['Value']
targetSE = StorageElement( target )
targetToken = targetSE.getStorageParameters( "SRM2" )
if not targetToken["OK"]:
log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) )
continue
# # create FTSJob
for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
ftsJob = FTSJob()
ftsJob.RequestID = request.RequestID
ftsJob.OperationID = operation.OperationID
ftsJob.SourceSE = source
ftsJob.TargetSE = target
ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
ftsJob.FTSServer = route.ftsServer
for ftsFile in fileList:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
if not submit["OK"]:
log.error( "unable to submit FTSJob:", submit["Message"] )
continue
log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in ftsJob:
ftsFile.FTSGUID = ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
# # update placement route
try:
self.updateLock().acquire()
self.__ftsPlacement.startTransferOnRoute( route )
finally:
self.updateLock().release()
ftsJobs.append( ftsJob )
log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
return S_OK( ftsJobs )
def __writeSEInfo(xml_doc, xml_site, site, site_tier, site_subtier):
""" Writes SE information in the XML Document
"""
def __write_SE_XML(site_se_opts):
"""Sub-function just to populate the XML with the SE values
"""
site_se_name = site_se_opts.get('Host')
site_se_flavour = site_se_opts.get('Protocol')
site_se_path = site_se_opts.get('Path', 'UNDEFINED')
site_se_endpoint = site_se_opts.get('URLBase')
mappingSEFlavour = {'srm': 'SRMv2',
'root': 'XROOTD', 'http': 'HTTPS'}
xml_se = xml_append(xml_doc, xml_site, 'service',
endpoint=site_se_endpoint,
flavour=mappingSEFlavour.get(site_se_flavour, 'UNDEFINED'),
hostname=site_se_name,
path=site_se_path)
# ipv6 status of the SE
i6Status = NagiosTopologyAgent.isHostIPV6(site_se_name)
i6Comment = ""
if i6Status == -1:
i6Comment = "Maybe DIRAC Service, not a valid machine"
xml_append(xml_doc, xml_se, 'queues', ipv6_status=str(i6Status), ipv6_comment=i6Comment)
has_grid_elem = True
real_site_name = site.split(".")[1]
dmsHelper = DMSHelpers()
if int(site_tier) in (0, 1):
dst = dmsHelper.getSEInGroupAtSite('Tier1-DST', real_site_name)
raw = dmsHelper.getSEInGroupAtSite('Tier1-RAW', real_site_name)
if not raw['OK']:
gLogger.error(raw['Message'])
return False
raw = raw['Value']
se_RAW = StorageElement(raw)
se_plugins_RAW = se_RAW.getPlugins()
if site_subtier == 'T2-D':
dst = dmsHelper.getSEInGroupAtSite('Tier2D-DST', real_site_name)
if not dst['OK']:
gLogger.error(dst['Message'])
return False
dst = dst['Value']
se_DST = StorageElement(dst)
se_plugins_DST = se_DST.getPlugins()
if not se_plugins_DST['OK']:
gLogger.error(se_plugins_DST['Message'])
return False
for protocol in se_plugins_DST['Value']:
site_se_opts_DST = se_DST.getStorageParameters(protocol)
if not site_se_opts_DST['OK']:
gLogger.error(site_se_opts_DST['Message'])
return False
site_se_opts_DST = site_se_opts_DST['Value']
__write_SE_XML(site_se_opts_DST)
if int(site_tier) in (0, 1):
if protocol in se_plugins_RAW['Value']:
site_se_opts_RAW = se_RAW.getStorageParameters(protocol)
if not site_se_opts_RAW['OK']:
gLogger.error(site_se_opts_RAW['Message'])
return has_grid_elem
site_se_opts_RAW = site_se_opts_RAW['Value']
# This tests if the DST and RAW StorageElements have the same endpoint.
# If so it only uses the one already added.
if site_se_opts_RAW['Host'] != site_se_opts_DST['Host']:
__write_SE_XML(site_se_opts_RAW)
return has_grid_elem
def __submit( self, request, operation, toSubmit ):
""" create and submit new FTSJobs using list of FTSFiles
:param Request request: ReqDB.Request instance
:param list ftsFiles: list of FTSFile instances
:return: [ FTSJob, FTSJob, ...]
"""
log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) )
bySourceAndTarget = {}
for ftsFile in toSubmit:
if ftsFile.SourceSE not in bySourceAndTarget:
bySourceAndTarget.setdefault( ftsFile.SourceSE, {} )
if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]:
bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] )
bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile )
ftsJobs = []
for source, targetDict in bySourceAndTarget.iteritems():
for target, ftsFileList in targetDict.iteritems():
log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) )
route = self.__ftsPlacement.findRoute( source, target )
if not route["OK"]:
log.error( route["Message"] )
continue
route = route["Value"]
routeValid = self.__ftsPlacement.isRouteValid( route )
if not routeValid['OK']:
log.error( "Route invalid : %s" % routeValid['Message'] )
continue
sourceSE = StorageElement( source )
sourceToken = sourceSE.getStorageParameters( protocol = 'srm' )
if not sourceToken["OK"]:
log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) )
continue
seStatus = sourceSE.getStatus()['Value']
targetSE = StorageElement( target )
targetToken = targetSE.getStorageParameters( protocol = 'srm' )
if not targetToken["OK"]:
log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) )
continue
# # create FTSJob
for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ):
ftsJob = FTSJob()
ftsJob.RequestID = request.RequestID
ftsJob.OperationID = operation.OperationID
ftsJob.SourceSE = source
ftsJob.TargetSE = target
ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" )
ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" )
ftsJob.FTSServer = route.ftsServer
for ftsFile in fileList:
ftsFile.Attempt += 1
ftsFile.Error = ""
ftsJob.addFile( ftsFile )
submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 )
if not submit["OK"]:
log.error( "unable to submit FTSJob:", submit["Message"] )
continue
log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) )
# # update statuses for job files
for ftsFile in ftsJob:
ftsFile.FTSGUID = ftsJob.FTSGUID
ftsFile.Status = "Submitted"
ftsFile.Attempt += 1
# # update placement route
try:
self.updateLock().acquire()
self.__ftsPlacement.startTransferOnRoute( route )
finally:
self.updateLock().release()
ftsJobs.append( ftsJob )
log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) )
return S_OK( ftsJobs )
def getStorageElements(vo):
"""
Get configuration of storage elements
:param vo: VO name that an SE supports
:return: S_OK/S_ERROR, Value dictionary with key SE and value protocol list
"""
log = gLogger.getLocalSubLogger("RucioSynchronizer/%s" % vo)
seProtocols = {}
dms = DMSHelpers(vo=vo)
for seName in dms.getStorageElements():
se = StorageElement(seName)
if not se.valid:
log.warn("Storage element is not valid.", seName)
continue
if vo not in se.options.get("VO", []):
log.debug("SE is valid, but it doesn't support the VO. Skipped.",
"[SE: %s, VO: %s]" % (seName, vo))
continue
log.debug(" Processing a valid SE for VO: ",
"[SE:%s, VO:%s]" % (seName, vo))
log.debug("Available SE options ", se.options)
seProtocols[seName] = []
all_protocols = []
read_protocols = {}
protocols = se.options.get("AccessProtocols")
log.debug("Global AccessProtocols:",
"[VO: %s, protocols: %s]" % (vo, protocols))
if not protocols:
protocols = dms.getAccessProtocols()
if not protocols:
log.warn(
" No global or SE specific access protocols defined for SE ",
seName)
continue
log.debug("AccessProtocols:",
"[VO: %s, protocols:%s]" % (vo, protocols))
idx = 1
for prot in protocols:
read_protocols[prot] = idx
idx += 1
if prot not in all_protocols:
all_protocols.append(prot)
write_protocols = {}
protocols = se.options.get("WriteProtocols")
if not protocols:
if not protocols:
protocols = dms.getWriteProtocols()
if not protocols:
log.warn(
" No global or SE specific write protocols defined for SE ",
seName)
continue
idx = 1
for prot in protocols:
write_protocols[prot] = idx
idx += 1
if prot not in all_protocols:
all_protocols.append(prot)
mapping = {
"Protocol": "scheme",
"Host": "hostname",
"Port": "port",
"Path": "prefix"
}
for protocol in all_protocols:
space_token = None
params = {
"hostname": None,
"scheme": None,
"port": None,
"prefix": None,
"impl": "rucio.rse.protocols.gfal.Default",
"domains": {
"lan": {
"read": 0,
"write": 0,
"delete": 0
},
"wan": {
"read": 0,
"write": 0,
"delete": 0,
"third_party_copy": 0
},
},
}
res = se.getStorageParameters(protocol=protocol)
if res["OK"]:
values = res["Value"]
for key in [
"Protocol", "Host", "Access", "Path", "Port", "WSUrl",
"SpaceToken", "WSUrl", "PluginName"
]:
value = values.get(key)
if key in mapping:
params[mapping[key]] = value
else:
if key == "SpaceToken":
space_token = value
if params["scheme"] == "srm" and key == "WSUrl":
params["extended_attributes"] = {
"web_service_path": "%s" % value,
"space_token": space_token,
}
if key == "Protocol":
params["domains"]["lan"]["read"] = read_protocols.get(
value, 0)
params["domains"]["wan"]["read"] = read_protocols.get(
value, 0)
params["domains"]["lan"][
"write"] = write_protocols.get(value, 0)
params["domains"]["wan"][
"write"] = write_protocols.get(value, 0)
params["domains"]["lan"][
"delete"] = write_protocols.get(value, 0)
params["domains"]["wan"][
"delete"] = write_protocols.get(value, 0)
params["domains"]["wan"][
"third_party_copy"] = write_protocols.get(
value, 0)
seProtocols[seName].append(params)
log.debug("Accepted Dirac SEs: ", seProtocols)
return S_OK(seProtocols)
