class ResourcesTestCase( unittest.TestCase ):
def setUp( self ):
Script.disableCS( )
Script.parseCommandLine()
self.resources = Resources()
def test_getSites( self ):
print
result = self.resources.getSites( {'Name':['CERN','CPPM','PNPI']} )
self.assertTrue( result['OK'], 'getSites' )
sites = result['Value']
print sites
result = self.resources.getEligibleSites( {'Name':['CERN','CPPM','PNPI']} )
self.assertTrue( result['OK'], 'getEligibleSites' )
eligibleSites = result['Value']
self.assertEqual(sites, eligibleSites, 'sites and eligible sites are the same')
def test_getResources( self ):
print
result = self.resources.getResources( 'CERN', 'Storage' )
self.assertTrue( result['OK'], 'getResources' )
ses = result['Value']
print ses
def test_getNodes( self ):
print
result = self.resources.getNodes( 'CERN::ce130', 'Queue' )
self.assertTrue( result['OK'], 'getNodes' )
nodes = result['Value']
print nodes
def test_getEligibleResources( self ):
print
result = self.resources.getEligibleResources( 'Computing', { 'Site':['CERN','CPPM','Zurich'],'SubmissionMode':'Direct' } )
self.assertTrue( result['OK'], 'getEligibleResources' )
ces = result['Value']
print ces
def test_getEligibleNodes( self ):
print
result = self.resources.getEligibleNodes( 'AccessProtocol',
{ 'Site':['CERN','CPPM','Zurich'] },
{ 'Protocol':'srm' } )
self.assertTrue( result['OK'], 'getEligibleNodes' )
aps = result['Value']
print aps
def test_getEligibleComputingElements( self ):
siteMask = ['LCG.CERN.ch','LCG.CPPM.fr']
result = self.resources.getEligibleResources( 'Computing', {'Site':siteMask,
'SubmissionMode':'gLite',
'CEType':['LCG','CREAM']} )
self.assertTrue( result['OK'], 'getEligibleResources' )
print
for ce in result['Value']:
ceHost = self.resources.getComputingElementValue( ce, 'Host', 'unknown' )
print ce, ceHost
def _resolveCECandidates( self, taskQueueDict ):
"""
Return a list of CEs for this TaskQueue
"""
# assume user knows what they're doing and avoid site mask e.g. sam jobs
if 'GridCEs' in taskQueueDict and taskQueueDict['GridCEs']:
self.log.info( 'CEs requested by TaskQueue %s:' % taskQueueDict['TaskQueueID'],
', '.join( taskQueueDict['GridCEs'] ) )
return taskQueueDict['GridCEs']
# Get the mask
siteStatus = SiteStatus()
ret = siteStatus.getUsableSites( 'ComputingAccess' )
if not ret['OK']:
self.log.error( 'Can not retrieve site Mask from DB:', ret['Message'] )
return []
usableSites = ret['Value']
if not usableSites:
self.log.error( 'Site mask is empty' )
return []
self.log.verbose( 'Site Mask: %s' % ', '.join( usableSites ) )
# remove banned sites from siteMask
if 'BannedSites' in taskQueueDict:
for site in taskQueueDict['BannedSites']:
if site in usableSites:
usableSites.remove( site )
self.log.verbose( 'Removing banned site %s from site Mask' % site )
# remove from the mask if a Site is given
siteMask = [ site for site in usableSites if 'Sites' not in taskQueueDict or site in taskQueueDict['Sites'] ]
if not siteMask:
# pilot can not be submitted
self.log.info( 'No Valid Site Candidate in Mask for TaskQueue %s' % taskQueueDict['TaskQueueID'] )
return []
self.log.info( 'Site Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( siteMask ) )
# Get CE's associates to the given site Names
ceMask = []
resources = Resources( vo = self.virtualOrganization )
result = resources.getEligibleResources( 'Computing', {'Site':siteMask,
'SubmissionMode':'gLite',
'CEType':['LCG','CREAM']} )
if not result['OK']:
self.log.error( "Failed to get eligible ce's:", result['Message'] )
return []
ces = result['Value']
for ce in ces:
ceHost = resources.getComputingElementValue( ce, 'Host', 'unknown' )
if ceHost != 'unknown':
ceMask.append( ceHost )
if not ceMask:
self.log.info( 'No CE Candidate found for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( siteMask ) )
self.log.verbose( 'CE Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'], ', '.join( ceMask ) )
return ceMask
def _resolveCECandidates(self, taskQueueDict):
"""
Return a list of CEs for this TaskQueue
"""
# assume user knows what they're doing and avoid site mask e.g. sam jobs
if 'GridCEs' in taskQueueDict and taskQueueDict['GridCEs']:
self.log.info(
'CEs requested by TaskQueue %s:' %
taskQueueDict['TaskQueueID'],
', '.join(taskQueueDict['GridCEs']))
return taskQueueDict['GridCEs']
# Get the mask
siteStatus = SiteStatus()
ret = siteStatus.getUsableSites('ComputingAccess')
if not ret['OK']:
self.log.error('Can not retrieve site Mask from DB:',
ret['Message'])
return []
usableSites = ret['Value']
if not usableSites:
self.log.error('Site mask is empty')
return []
self.log.verbose('Site Mask: %s' % ', '.join(usableSites))
# remove banned sites from siteMask
if 'BannedSites' in taskQueueDict:
for site in taskQueueDict['BannedSites']:
if site in usableSites:
usableSites.remove(site)
self.log.verbose('Removing banned site %s from site Mask' %
site)
# remove from the mask if a Site is given
siteMask = [
site for site in usableSites
if 'Sites' not in taskQueueDict or site in taskQueueDict['Sites']
]
if not siteMask:
# pilot can not be submitted
self.log.info('No Valid Site Candidate in Mask for TaskQueue %s' %
taskQueueDict['TaskQueueID'])
return []
self.log.info(
'Site Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'],
', '.join(siteMask))
# Get CE's associates to the given site Names
ceMask = []
resources = Resources(vo=self.virtualOrganization)
result = resources.getEligibleResources(
'Computing', {
'Site': siteMask,
'SubmissionMode': 'gLite',
'CEType': ['LCG', 'CREAM']
})
if not result['OK']:
self.log.error("Failed to get eligible ce's:", result['Message'])
return []
ces = result['Value']
for ce in ces:
ceHost = resources.getComputingElementValue(ce, 'Host', 'unknown')
if ceHost != 'unknown':
ceMask.append(ceHost)
if not ceMask:
self.log.info(
'No CE Candidate found for TaskQueue %s:' %
taskQueueDict['TaskQueueID'], ', '.join(siteMask))
self.log.verbose(
'CE Candidates for TaskQueue %s:' % taskQueueDict['TaskQueueID'],
', '.join(ceMask))
return ceMask
def __lookForCE( self ):
knownces = self.am_getOption( 'BannedCEs', [] )
resources = Resources( self.voName )
result = resources.getEligibleResources( 'Computing', {'CEType':['LCG','CREAM'] } )
if not result['OK']:
return result
knownces = [ resources.getComputingElementValue( x, 'Host' ) for x in result['Value'] ]
# result = gConfig.getSections( '/Resources/Sites' )
# if not result['OK']:
# return
# grids = result['Value']
#
# for grid in grids:
#
# result = gConfig.getSections( '/Resources/Sites/%s' % grid )
# if not result['OK']:
# return
# sites = result['Value']
#
# for site in sites:
# opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
# ces = List.fromChar( opt.get( 'CE', '' ) )
# knownces += ces
response = ldapCEState( '', vo = self.voName )
if not response['OK']:
self.log.error( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName )
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split( ":" )[0]
if not cename in knownces:
newces[cename] = None
self.log.debug( "newce", cename )
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCluster, ce )
continue
clusters = response['Value']
if len( clusters ) != 1:
self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) )
if len( clusters ) == 0:
continue
cluster = clusters[0]
fkey = cluster.get( 'GlueForeignKey', [] )
if type( fkey ) == type( '' ):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count( 'GlueSiteUniqueID' ):
nameBDII = entry.split( '=' )[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII )
self.log.info( cestring )
response = ldapCE( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
ceinfos = response['Value']
if len( ceinfos ):
ceinfo = ceinfos[0]
systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease )
self.log.info( osstring )
response = ldapCEState( ce, vo = self.voName )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
newcestring = "\n\n%s\n%s" % ( cestring, osstring )
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' )
queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' )
queuestring = "%s %s" % ( queuename, queuestatus )
self.log.info( queuestring )
newcestring += "\n%s" % queuestring
if queuestatus.count( 'Production' ):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) )
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % ( body, possibleNewSite )
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return S_OK()
class ResourcesTestCase(unittest.TestCase):
def setUp(self):
Script.disableCS()
Script.parseCommandLine()
self.resources = Resources()
def test_getSites(self):
print
result = self.resources.getSites({'Name': ['CERN', 'CPPM', 'PNPI']})
self.assertTrue(result['OK'], 'getSites')
sites = result['Value']
print sites
result = self.resources.getEligibleSites(
{'Name': ['CERN', 'CPPM', 'PNPI']})
self.assertTrue(result['OK'], 'getEligibleSites')
eligibleSites = result['Value']
self.assertEqual(sites, eligibleSites,
'sites and eligible sites are the same')
def test_getResources(self):
print
result = self.resources.getResources('CERN', 'Storage')
self.assertTrue(result['OK'], 'getResources')
ses = result['Value']
print ses
def test_getNodes(self):
print
result = self.resources.getNodes('CERN::ce130', 'Queue')
self.assertTrue(result['OK'], 'getNodes')
nodes = result['Value']
print nodes
def test_getEligibleResources(self):
print
result = self.resources.getEligibleResources(
'Computing', {
'Site': ['CERN', 'CPPM', 'Zurich'],
'SubmissionMode': 'Direct'
})
self.assertTrue(result['OK'], 'getEligibleResources')
ces = result['Value']
print ces
def test_getEligibleNodes(self):
print
result = self.resources.getEligibleNodes(
'AccessProtocol', {'Site': ['CERN', 'CPPM', 'Zurich']},
{'Protocol': 'srm'})
self.assertTrue(result['OK'], 'getEligibleNodes')
aps = result['Value']
print aps
def test_getEligibleComputingElements(self):
siteMask = ['LCG.CERN.ch', 'LCG.CPPM.fr']
result = self.resources.getEligibleResources(
'Computing', {
'Site': siteMask,
'SubmissionMode': 'gLite',
'CEType': ['LCG', 'CREAM']
})
self.assertTrue(result['OK'], 'getEligibleResources')
print
for ce in result['Value']:
ceHost = self.resources.getComputingElementValue(
ce, 'Host', 'unknown')
print ce, ceHost