def doNew(self, masterParams=None): """ Gets the total and the free disk space of a DIPS storage element that is found in the CS and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. """ if masterParams is not None: elementName = masterParams else: elementName = self._prepareCommand() if not elementName['OK']: return elementName se = StorageElement(elementName) elementURL = se.getStorageParameters(protocol="dips") if elementURL['OK']: elementURL = se.getStorageParameters( protocol="dips")['Value']['URLBase'] else: gLogger.verbose("Not a DIPS storage element, skipping...") return S_OK() self.rpc = RPCClient(elementURL, timeout=120) free = self.rpc.getFreeDiskSpace("/") if not free['OK']: return free free = free['Value'] total = self.rpc.getTotalDiskSpace("/") if not total['OK']: return total total = total['Value'] if free and free < 1: free = 1 if total and total < 1: total = 1 result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName) if not result['OK']: return result return S_OK()
def doNew(self, masterParams=None): """ Gets the total and the free disk space of a DIPS storage element that is found in the CS and inserts the results in the SpaceTokenOccupancyCache table of ResourceManagementDB database. """ if masterParams is not None: elementName = masterParams else: elementName = self._prepareCommand() if not elementName["OK"]: return elementName se = StorageElement(elementName) elementURL = se.getStorageParameters(protocol="dips") if elementURL["OK"]: elementURL = se.getStorageParameters(protocol="dips")["Value"]["URLBase"] else: gLogger.verbose("Not a DIPS storage element, skipping...") return S_OK() self.rpc = RPCClient(elementURL, timeout=120) free = self.rpc.getFreeDiskSpace("/") if not free["OK"]: return free free = free["Value"] total = self.rpc.getTotalDiskSpace("/") if not total["OK"]: return total total = total["Value"] if free and free < 1: free = 1 if total and total < 1: total = 1 result = self.rsClient.addOrModifySpaceTokenOccupancyCache( endpoint=elementURL, lastCheckTime=datetime.utcnow(), free=free, total=total, token=elementName ) if not result["OK"]: return result return S_OK()
def getSEParameters(seName): """get all the SE parameters in a list :param str seName: name of the Storage Element :return: S_OK() with list of dict with parameters """ # This import is here to avoid circular imports from DIRAC.Resources.Storage.StorageElement import StorageElement se = StorageElement(seName, hideExceptions=True) protocolsSet = set(se.localAccessProtocolList) | set( se.localWriteProtocolList) seParametersList = [] for protocol in protocolsSet: seParameters = se.getStorageParameters(protocol=protocol) if seParameters["OK"]: seParametersList.append(seParameters["Value"]) else: gLogger.verbose("No SE parameters obtained", "for SE %s and protocol %s" % (seName, protocol)) return S_OK(seParametersList)
def _getSEParameters(seName): se = StorageElement(seName, hideExceptions=True) pluginsList = se.getPlugins() if not pluginsList['OK']: gLogger.warn(pluginsList['Message'], "SE: %s" % seName) return pluginsList pluginsList = pluginsList['Value'] if 'SRM2' in pluginsList: pluginsList.remove('SRM2') pluginsList.insert(0, 'SRM2') for plugin in pluginsList: seParameters = se.getStorageParameters(plugin) if seParameters['OK']: break return seParameters
def _getSEParameters( seName ): se = StorageElement( seName, hideExceptions = True ) pluginsList = se.getPlugins() if not pluginsList['OK']: gLogger.warn( pluginsList['Message'], "SE: %s" % seName ) return pluginsList pluginsList = pluginsList['Value'] if 'SRM2' in pluginsList: pluginsList.remove( 'SRM2' ) pluginsList.insert( 0, 'SRM2' ) for plugin in pluginsList: seParameters = se.getStorageParameters( plugin ) if seParameters['OK']: break return seParameters
def __fetchSpaceToken(seName): """ Fetch the space token of storage element :param seName name of the storageElement :returns space token """ seToken = None if seName: seObj = StorageElement(seName) res = seObj.getStorageParameters(protocol='srm') if not res['OK']: return res seToken = res["Value"].get("SpaceToken") return S_OK(seToken)
def _getSEParameters(seName): se = StorageElement(seName, hideExceptions=True) pluginsList = se.getPlugins() if not pluginsList['OK']: gLogger.warn(pluginsList['Message'], "SE: %s" % seName) return pluginsList pluginsList = pluginsList['Value'] # Put the srm capable protocol first, but why doing that is a # mystery that will eventually need to be sorted out... for plugin in ('GFAL2_SRM2', 'SRM2'): if plugin in pluginsList: pluginsList.remove(plugin) pluginsList.insert(0, plugin) for plugin in pluginsList: seParameters = se.getStorageParameters(plugin) if seParameters['OK']: break return seParameters
def _getSEParameters( seName ): se = StorageElement( seName, hideExceptions = True ) pluginsList = se.getPlugins() if not pluginsList['OK']: gLogger.warn( pluginsList['Message'], "SE: %s" % seName ) return pluginsList pluginsList = pluginsList['Value'] # Put the srm capable protocol first, but why doing that is a # mystery that will eventually need to be sorted out... for plugin in ( 'GFAL2_SRM2', 'SRM2' ): if plugin in pluginsList: pluginsList.remove( plugin ) pluginsList.insert( 0, plugin ) for plugin in pluginsList: seParameters = se.getStorageParameters( plugin ) if seParameters['OK']: break return seParameters
def __fetchSpaceToken(seName): """ Fetch the space token of storage element :param seName name of the storageElement :returns space token. If there is no SpaceToken defined, returns None """ seToken = None if seName: seObj = StorageElement(seName) res = seObj.getStorageParameters(protocol='srm') if not res['OK']: # If there is no SRM protocol, we do not specify # the space token if cmpError(res, errno.ENOPROTOOPT): return S_OK(None) return res seToken = res["Value"].get("SpaceToken") return S_OK(seToken)
def __submit( self, request, operation, toSubmit ): """ create and submit new FTSJobs using list of FTSFiles :param Request request: ReqDB.Request instance :param list ftsFiles: list of FTSFile instances :return: [ FTSJob, FTSJob, ...] """ log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) ) bySourceAndTarget = {} for ftsFile in toSubmit: if ftsFile.SourceSE not in bySourceAndTarget: bySourceAndTarget.setdefault( ftsFile.SourceSE, {} ) if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]: bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] ) bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile ) ftsJobs = [] for source, targetDict in bySourceAndTarget.iteritems(): for target, ftsFileList in targetDict.iteritems(): log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) ) route = self.__ftsPlacement.findRoute( source, target ) if not route["OK"]: log.error( route["Message"] ) continue route = route["Value"] routeValid = self.__ftsPlacement.isRouteValid( route ) if not routeValid['OK']: log.error( "Route invalid : %s" % routeValid['Message'] ) continue sourceSE = StorageElement( source ) sourceToken = sourceSE.getStorageParameters( "SRM2" ) if not sourceToken["OK"]: log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) ) continue seStatus = sourceSE.getStatus()['Value'] targetSE = StorageElement( target ) targetToken = targetSE.getStorageParameters( "SRM2" ) if not targetToken["OK"]: log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) ) continue # # create FTSJob for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ): ftsJob = FTSJob() ftsJob.RequestID = request.RequestID ftsJob.OperationID = operation.OperationID ftsJob.SourceSE = source ftsJob.TargetSE = target ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" ) ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" ) ftsJob.FTSServer = route.ftsServer for ftsFile in fileList: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 ) if not submit["OK"]: log.error( "unable to submit FTSJob:", submit["Message"] ) continue log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) ) # # update statuses for job files for ftsFile in ftsJob: ftsFile.FTSGUID = ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 # # update placement route try: self.updateLock().acquire() self.__ftsPlacement.startTransferOnRoute( route ) finally: self.updateLock().release() ftsJobs.append( ftsJob ) log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) ) return S_OK( ftsJobs )
def __writeSEInfo(xml_doc, xml_site, site, site_tier, site_subtier): """ Writes SE information in the XML Document """ def __write_SE_XML(site_se_opts): """Sub-function just to populate the XML with the SE values """ site_se_name = site_se_opts.get('Host') site_se_flavour = site_se_opts.get('Protocol') site_se_path = site_se_opts.get('Path', 'UNDEFINED') site_se_endpoint = site_se_opts.get('URLBase') mappingSEFlavour = {'srm': 'SRMv2', 'root': 'XROOTD', 'http': 'HTTPS'} xml_se = xml_append(xml_doc, xml_site, 'service', endpoint=site_se_endpoint, flavour=mappingSEFlavour.get(site_se_flavour, 'UNDEFINED'), hostname=site_se_name, path=site_se_path) # ipv6 status of the SE i6Status = NagiosTopologyAgent.isHostIPV6(site_se_name) i6Comment = "" if i6Status == -1: i6Comment = "Maybe DIRAC Service, not a valid machine" xml_append(xml_doc, xml_se, 'queues', ipv6_status=str(i6Status), ipv6_comment=i6Comment) has_grid_elem = True real_site_name = site.split(".")[1] dmsHelper = DMSHelpers() if int(site_tier) in (0, 1): dst = dmsHelper.getSEInGroupAtSite('Tier1-DST', real_site_name) raw = dmsHelper.getSEInGroupAtSite('Tier1-RAW', real_site_name) if not raw['OK']: gLogger.error(raw['Message']) return False raw = raw['Value'] se_RAW = StorageElement(raw) se_plugins_RAW = se_RAW.getPlugins() if site_subtier == 'T2-D': dst = dmsHelper.getSEInGroupAtSite('Tier2D-DST', real_site_name) if not dst['OK']: gLogger.error(dst['Message']) return False dst = dst['Value'] se_DST = StorageElement(dst) se_plugins_DST = se_DST.getPlugins() if not se_plugins_DST['OK']: gLogger.error(se_plugins_DST['Message']) return False for protocol in se_plugins_DST['Value']: site_se_opts_DST = se_DST.getStorageParameters(protocol) if not site_se_opts_DST['OK']: gLogger.error(site_se_opts_DST['Message']) return False site_se_opts_DST = site_se_opts_DST['Value'] __write_SE_XML(site_se_opts_DST) if int(site_tier) in (0, 1): if protocol in se_plugins_RAW['Value']: site_se_opts_RAW = se_RAW.getStorageParameters(protocol) if not site_se_opts_RAW['OK']: gLogger.error(site_se_opts_RAW['Message']) return has_grid_elem site_se_opts_RAW = site_se_opts_RAW['Value'] # This tests if the DST and RAW StorageElements have the same endpoint. # If so it only uses the one already added. if site_se_opts_RAW['Host'] != site_se_opts_DST['Host']: __write_SE_XML(site_se_opts_RAW) return has_grid_elem
def __submit( self, request, operation, toSubmit ): """ create and submit new FTSJobs using list of FTSFiles :param Request request: ReqDB.Request instance :param list ftsFiles: list of FTSFile instances :return: [ FTSJob, FTSJob, ...] """ log = self.log.getSubLogger( "req_%s/%s/submit" % ( request.RequestID, request.RequestName ) ) bySourceAndTarget = {} for ftsFile in toSubmit: if ftsFile.SourceSE not in bySourceAndTarget: bySourceAndTarget.setdefault( ftsFile.SourceSE, {} ) if ftsFile.TargetSE not in bySourceAndTarget[ftsFile.SourceSE]: bySourceAndTarget[ftsFile.SourceSE].setdefault( ftsFile.TargetSE, [] ) bySourceAndTarget[ftsFile.SourceSE][ftsFile.TargetSE].append( ftsFile ) ftsJobs = [] for source, targetDict in bySourceAndTarget.iteritems(): for target, ftsFileList in targetDict.iteritems(): log.info( "found %s files to submit from %s to %s" % ( len( ftsFileList ), source, target ) ) route = self.__ftsPlacement.findRoute( source, target ) if not route["OK"]: log.error( route["Message"] ) continue route = route["Value"] routeValid = self.__ftsPlacement.isRouteValid( route ) if not routeValid['OK']: log.error( "Route invalid : %s" % routeValid['Message'] ) continue sourceSE = StorageElement( source ) sourceToken = sourceSE.getStorageParameters( protocol = 'srm' ) if not sourceToken["OK"]: log.error( "unable to get sourceSE parameters:", "(%s) %s" % ( source, sourceToken["Message"] ) ) continue seStatus = sourceSE.getStatus()['Value'] targetSE = StorageElement( target ) targetToken = targetSE.getStorageParameters( protocol = 'srm' ) if not targetToken["OK"]: log.error( "unable to get targetSE parameters:", "(%s) %s" % ( target, targetToken["Message"] ) ) continue # # create FTSJob for fileList in breakListIntoChunks( ftsFileList, self.MAX_FILES_PER_JOB ): ftsJob = FTSJob() ftsJob.RequestID = request.RequestID ftsJob.OperationID = operation.OperationID ftsJob.SourceSE = source ftsJob.TargetSE = target ftsJob.SourceToken = sourceToken["Value"].get( "SpaceToken", "" ) ftsJob.TargetToken = targetToken["Value"].get( "SpaceToken", "" ) ftsJob.FTSServer = route.ftsServer for ftsFile in fileList: ftsFile.Attempt += 1 ftsFile.Error = "" ftsJob.addFile( ftsFile ) submit = ftsJob.submitFTS( self.__ftsVersion, command = self.SUBMIT_COMMAND, pinTime = self.PIN_TIME if seStatus['TapeSE'] else 0 ) if not submit["OK"]: log.error( "unable to submit FTSJob:", submit["Message"] ) continue log.info( "FTSJob '%s'@'%s' has been submitted" % ( ftsJob.FTSGUID, ftsJob.FTSServer ) ) # # update statuses for job files for ftsFile in ftsJob: ftsFile.FTSGUID = ftsJob.FTSGUID ftsFile.Status = "Submitted" ftsFile.Attempt += 1 # # update placement route try: self.updateLock().acquire() self.__ftsPlacement.startTransferOnRoute( route ) finally: self.updateLock().release() ftsJobs.append( ftsJob ) log.info( "%s new FTSJobs have been submitted" % len( ftsJobs ) ) return S_OK( ftsJobs )
def getStorageElements(vo): """ Get configuration of storage elements :param vo: VO name that an SE supports :return: S_OK/S_ERROR, Value dictionary with key SE and value protocol list """ log = gLogger.getLocalSubLogger("RucioSynchronizer/%s" % vo) seProtocols = {} dms = DMSHelpers(vo=vo) for seName in dms.getStorageElements(): se = StorageElement(seName) if not se.valid: log.warn("Storage element is not valid.", seName) continue if vo not in se.options.get("VO", []): log.debug("SE is valid, but it doesn't support the VO. Skipped.", "[SE: %s, VO: %s]" % (seName, vo)) continue log.debug(" Processing a valid SE for VO: ", "[SE:%s, VO:%s]" % (seName, vo)) log.debug("Available SE options ", se.options) seProtocols[seName] = [] all_protocols = [] read_protocols = {} protocols = se.options.get("AccessProtocols") log.debug("Global AccessProtocols:", "[VO: %s, protocols: %s]" % (vo, protocols)) if not protocols: protocols = dms.getAccessProtocols() if not protocols: log.warn( " No global or SE specific access protocols defined for SE ", seName) continue log.debug("AccessProtocols:", "[VO: %s, protocols:%s]" % (vo, protocols)) idx = 1 for prot in protocols: read_protocols[prot] = idx idx += 1 if prot not in all_protocols: all_protocols.append(prot) write_protocols = {} protocols = se.options.get("WriteProtocols") if not protocols: if not protocols: protocols = dms.getWriteProtocols() if not protocols: log.warn( " No global or SE specific write protocols defined for SE ", seName) continue idx = 1 for prot in protocols: write_protocols[prot] = idx idx += 1 if prot not in all_protocols: all_protocols.append(prot) mapping = { "Protocol": "scheme", "Host": "hostname", "Port": "port", "Path": "prefix" } for protocol in all_protocols: space_token = None params = { "hostname": None, "scheme": None, "port": None, "prefix": None, "impl": "rucio.rse.protocols.gfal.Default", "domains": { "lan": { "read": 0, "write": 0, "delete": 0 }, "wan": { "read": 0, "write": 0, "delete": 0, "third_party_copy": 0 }, }, } res = se.getStorageParameters(protocol=protocol) if res["OK"]: values = res["Value"] for key in [ "Protocol", "Host", "Access", "Path", "Port", "WSUrl", "SpaceToken", "WSUrl", "PluginName" ]: value = values.get(key) if key in mapping: params[mapping[key]] = value else: if key == "SpaceToken": space_token = value if params["scheme"] == "srm" and key == "WSUrl": params["extended_attributes"] = { "web_service_path": "%s" % value, "space_token": space_token, } if key == "Protocol": params["domains"]["lan"]["read"] = read_protocols.get( value, 0) params["domains"]["wan"]["read"] = read_protocols.get( value, 0) params["domains"]["lan"][ "write"] = write_protocols.get(value, 0) params["domains"]["wan"][ "write"] = write_protocols.get(value, 0) params["domains"]["lan"][ "delete"] = write_protocols.get(value, 0) params["domains"]["wan"][ "delete"] = write_protocols.get(value, 0) params["domains"]["wan"][ "third_party_copy"] = write_protocols.get( value, 0) seProtocols[seName].append(params) log.debug("Accepted Dirac SEs: ", seProtocols) return S_OK(seProtocols)