def _prepareCommand( self ): ''' DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] if 'element' not in self.args: return S_ERROR( '"element" not found in self.args' ) element = self.args[ 'element' ] if 'elementType' not in self.args: return S_ERROR( '"elementType" not found in self.args' ) elementType = self.args[ 'elementType' ] if not element in [ 'Site', 'Resource' ]: return S_ERROR( 'element is not Site nor Resource' ) hours = None if 'hours' in self.args: hours = self.args[ 'hours' ] gocdbServiceType = None # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName( elementName ) if not gocSite[ 'OK' ]: return gocSite elementName = gocSite[ 'Value' ] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': # We need to distinguish if it's tape or disk if getStorageElementOptions( elementName )['Value']['TapeSE']: gocdbServiceType = "srm" elif getStorageElementOptions( elementName )['Value']['DiskSE']: gocdbServiceType = "srm.nearline" seHost = CSHelpers.getSEHost( elementName ) if not seHost: return S_ERROR( 'No seHost for %s' % elementName ) elementName = seHost return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def _prepareCommand( self ): ''' DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] if 'element' not in self.args: return S_ERROR( '"element" not found in self.args' ) element = self.args[ 'element' ] if 'elementType' not in self.args: return S_ERROR( '"elementType" not found in self.args' ) elementType = self.args[ 'elementType' ] if not element in [ 'Site', 'Resource' ]: return S_ERROR( 'element is not Site nor Resource' ) hours = None if 'hours' in self.args: hours = self.args[ 'hours' ] gocdbServiceType = None # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName( elementName ) if not gocSite[ 'OK' ]: return gocSite elementName = gocSite[ 'Value' ] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': # We need to distinguish if it's tape or disk if getStorageElementOptions( elementName )['Value']['TapeSE']: gocdbServiceType = "srm.nearline" elif getStorageElementOptions( elementName )['Value']['DiskSE']: gocdbServiceType = "srm" seHost = CSHelpers.getSEHost( elementName ) if not seHost: return S_ERROR( 'No seHost for %s' % elementName ) elementName = seHost return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def _prepareCommand(self): ''' DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args['name'] if 'element' not in self.args: return S_ERROR('"element" not found in self.args') element = self.args['element'] if 'elementType' not in self.args: return S_ERROR('"elementType" not found in self.args') elementType = self.args['elementType'] if not element in ['Site', 'Resource']: return S_ERROR('element is neither Site nor Resource') hours = None if 'hours' in self.args: hours = self.args['hours'] gOCDBServiceType = None # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName(elementName) if not gocSite[ 'OK']: # The site is most probably not a grid site - not an issue, of course pass # so, elementName remains unchanged else: elementName = gocSite['Value'] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': # We need to distinguish if it's tape or disk seOptions = getStorageElementOptions(elementName) if not seOptions['OK']: return seOptions if seOptions['Value'].get('TapeSE'): gOCDBServiceType = "srm.nearline" elif seOptions['Value'].get('DiskSE'): gOCDBServiceType = "srm" seHost = CSHelpers.getSEHost(elementName) if not seHost['OK']: return seHost seHost = seHost['Value'] if not seHost: return S_ERROR('No seHost for %s' % elementName) elementName = seHost elif elementType in ['FTS', 'FTS3']: gOCDBServiceType = 'FTS' try: #WARNING: this method presupposes that the server is an FTS3 type elementName = getGOCFTSName(elementName) except: return S_ERROR( 'No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)' ) return S_OK((element, elementName, hours, gOCDBServiceType))
def _prepareCommand( self ): ''' DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. ''' if 'name' not in self.args: return S_ERROR( '"name" not found in self.args' ) elementName = self.args[ 'name' ] if 'element' not in self.args: return S_ERROR( '"element" not found in self.args' ) element = self.args[ 'element' ] if 'elementType' not in self.args: return S_ERROR( '"elementType" not found in self.args' ) elementType = self.args[ 'elementType' ] if not element in [ 'Site', 'Resource' ]: return S_ERROR( 'element is neither Site nor Resource' ) hours = None if 'hours' in self.args: hours = self.args[ 'hours' ] gocdbServiceType = None # Transform DIRAC site names into GOCDB topics if element == 'Site': gocSite = getGOCSiteName( elementName ) if not gocSite[ 'OK' ]: return gocSite elementName = gocSite[ 'Value' ] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == 'StorageElement': # We need to distinguish if it's tape or disk seOptions = getStorageElementOptions( elementName ) if not seOptions['OK']: return seOptions if seOptions['Value'].get( 'TapeSE' ): gocdbServiceType = "srm.nearline" elif seOptions['Value'].get( 'DiskSE' ): gocdbServiceType = "srm" seHost = CSHelpers.getSEHost( elementName ) if not seHost['OK']: return seHost seHost = seHost['Value'] if not seHost: return S_ERROR( 'No seHost for %s' % elementName ) elementName = seHost elif elementType == 'FTS' or elementType == 'FTS3': gocdbServiceType = 'FTS' try: #WARNING: this method presupposes that the server is an FTS3 type elementName = getGOCFTSName(elementName) except: return S_ERROR( 'No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)' ) return S_OK( ( element, elementName, hours, gocdbServiceType ) )
def __getSiteCandidates( self, inputData ): """This method returns a list of possible site candidates based on the job input data requirement. For each site candidate, the number of files on disk and tape is resolved. """ fileSEs = {} for lfn, replicas in inputData.items(): siteList = [] for se in replicas.keys(): sites = self.__getSitesForSE( se ) if sites['OK']: siteList += sites['Value'] fileSEs[lfn] = uniqueElements( siteList ) siteCandidates = [] i = 0 for _fileName, sites in fileSEs.items(): if not i: siteCandidates = sites else: tempSite = [] for site in siteCandidates: if site in sites: tempSite.append( site ) siteCandidates = tempSite i += 1 if not len( siteCandidates ): return S_ERROR( 'No candidate sites available' ) #In addition, check number of files on tape and disk for each site #for optimizations during scheduling siteResult = {} for site in siteCandidates: siteResult[site] = { 'disk': [], 'tape': [] } seDict = {} for lfn, replicas in inputData.items(): for se in replicas.keys(): if se not in seDict: sites = self.__getSitesForSE( se ) if not sites['OK']: continue try: #storageElement = StorageElement( se ) result = self.resourceStatus.getStorageElementStatus( se, statusType = 'ReadAccess' ) if not result['OK']: continue seDict[se] = { 'Sites': sites['Value'], 'SEParams': result['Value'][se] } result = getStorageElementOptions( se ) if not result['OK']: continue seDict[se]['SEParams'].update(result['Value']) except Exception: self.log.exception( 'Failed to instantiate StorageElement( %s )' % se ) continue for site in seDict[se]['Sites']: if site in siteCandidates: if seDict[se]['SEParams']['ReadAccess'] and seDict[se]['SEParams']['DiskSE']: if lfn not in siteResult[site]['disk']: siteResult[site]['disk'].append( lfn ) if lfn in siteResult[site]['tape']: siteResult[site]['tape'].remove( lfn ) if seDict[se]['SEParams']['ReadAccess'] and seDict[se]['SEParams']['TapeSE']: if lfn not in siteResult[site]['tape'] and lfn not in siteResult[site]['disk']: siteResult[site]['tape'].append( lfn ) for site in siteResult: siteResult[site]['disk'] = len( siteResult[site]['disk'] ) siteResult[site]['tape'] = len( siteResult[site]['tape'] ) return S_OK( siteResult )
def __getSiteCandidates( self, inputData ): """This method returns a list of possible site candidates based on the job input data requirement. For each site candidate, the number of files on disk and tape is resolved. """ fileSEs = {} for lfn, replicas in inputData.items(): siteList = [] for se in replicas.keys(): sites = self.__getSitesForSE( se ) if sites['OK']: siteList += sites['Value'] fileSEs[lfn] = uniqueElements( siteList ) siteCandidates = [] i = 0 for fileName, sites in fileSEs.items(): if not i: siteCandidates = sites else: tempSite = [] for site in siteCandidates: if site in sites: tempSite.append( site ) siteCandidates = tempSite i += 1 if not len( siteCandidates ): return S_ERROR( 'No candidate sites available' ) #In addition, check number of files on tape and disk for each site #for optimizations during scheduling siteResult = {} for site in siteCandidates: siteResult[site] = { 'disk': [], 'tape': [] } seDict = {} for lfn, replicas in inputData.items(): for se in replicas.keys(): if se not in seDict: sites = self.__getSitesForSE( se ) if not sites['OK']: continue try: #storageElement = StorageElement( se ) result = self.resourceStatus.getStorageElementStatus( se, statusType = 'Read' ) if not result['OK']: continue seDict[se] = { 'Sites': sites['Value'], 'SEParams': result['Value'][se] } result = getStorageElementOptions( se ) if not result['OK']: continue seDict[se]['SEParams'].update(result['Value']) except Exception: self.log.exception( 'Failed to instantiate StorageElement( %s )' % se ) continue for site in seDict[se]['Sites']: if site in siteCandidates: if seDict[se]['SEParams']['Read'] and seDict[se]['SEParams']['DiskSE']: if lfn not in siteResult[site]['disk']: siteResult[site]['disk'].append( lfn ) if lfn in siteResult[site]['tape']: siteResult[site]['tape'].remove( lfn ) if seDict[se]['SEParams']['Read'] and seDict[se]['SEParams']['TapeSE']: if lfn not in siteResult[site]['tape'] and lfn not in siteResult[site]['disk']: siteResult[site]['tape'].append( lfn ) for site in siteResult: siteResult[site]['disk'] = len( siteResult[site]['disk'] ) siteResult[site]['tape'] = len( siteResult[site]['tape'] ) return S_OK( siteResult )
def _prepareCommand(self): """ DowntimeCommand requires four arguments: - name : <str> - element : Site / Resource - elementType: <str> If the elements are Site(s), we need to get their GOCDB names. They may not have, so we ignore them if they do not have. """ if "name" not in self.args: return S_ERROR('"name" not found in self.args') elementName = self.args["name"] if "element" not in self.args: return S_ERROR('"element" not found in self.args') element = self.args["element"] if "elementType" not in self.args: return S_ERROR('"elementType" not found in self.args') elementType = self.args["elementType"] if not element in ["Site", "Resource"]: return S_ERROR("element is neither Site nor Resource") hours = None if "hours" in self.args: hours = self.args["hours"] gocdbServiceType = None # Transform DIRAC site names into GOCDB topics if element == "Site": gocSite = getGOCSiteName(elementName) if not gocSite["OK"]: return gocSite elementName = gocSite["Value"] # The DIRAC se names mean nothing on the grid, but their hosts do mean. elif elementType == "StorageElement": # We need to distinguish if it's tape or disk seOptions = getStorageElementOptions(elementName) if not seOptions["OK"]: return seOptions if seOptions["Value"].get("TapeSE"): gocdbServiceType = "srm.nearline" elif seOptions["Value"].get("DiskSE"): gocdbServiceType = "srm" seHost = CSHelpers.getSEHost(elementName) if not seHost["OK"]: return seHost seHost = seHost["Value"] if not seHost: return S_ERROR("No seHost for %s" % elementName) elementName = seHost elif elementType in ["FTS", "FTS3"]: gocdbServiceType = "FTS" try: # WARNING: this method presupposes that the server is an FTS3 type elementName = getGOCFTSName(elementName) except: return S_ERROR("No FTS3 server specified in dirac.cfg (see Resources/FTSEndpoints)") return S_OK((element, elementName, hours, gocdbServiceType))