def _syncSites(self):
"""
Sync DB content with sites that are in the CS
"""
# sites in the DB now
sitesIn = self.rsDB.getMonitoredsList('Site', paramsList=['SiteName'])
sitesIn = [s[0] for s in sitesIn]
# sites in CS now
sitesList = getSites()['Value']
try:
sitesList.remove('LCG.Dummy.ch')
except ValueError:
pass
# remove sites from the DB not more in the CS
for site in sitesIn:
if site not in sitesList:
self.rsDB.removeSite(site)
# add to DB what is in CS now and wasn't before
for site in sitesList:
if site not in sitesIn:
# DIRAC Tier
tier = getSiteTier(site)['Value'][0]
if tier == 0 or tier == '0':
t = 'T0'
elif tier == 1 or tier == '1':
t = 'T1'
elif tier == 3 or tier == '3':
t = 'T3'
else:
t = 'T2'
#Grid Name of the site
gridSiteName = getGOCSiteName(site)
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
#Grid Tier (with a workaround!)
DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
if not DIRACSitesOfGridSites['OK']:
raise RSSException, DIRACSitesOfGridSites['Message']
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len(DIRACSitesOfGridSites) == 1:
gt = t
else:
gt = self.__getGOCTier(DIRACSitesOfGridSites)
self.rsDB.addOrModifySite(
site, t, gridSiteName, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
self.rsDB.addOrModifyGridSite(gridSiteName, gt)
sitesIn.append(site)
def _syncSites( self ):
"""
Sync DB content with sites that are in the CS
"""
# sites in the DB now
sitesIn = self.rsDB.getMonitoredsList( 'Site', paramsList = ['SiteName'] )
sitesIn = [s[0] for s in sitesIn]
# sites in CS now
sitesList = getSites()['Value']
try:
sitesList.remove( 'LCG.Dummy.ch' )
except ValueError:
pass
# remove sites from the DB not more in the CS
for site in sitesIn:
if site not in sitesList:
self.rsDB.removeSite( site )
# add to DB what is in CS now and wasn't before
for site in sitesList:
if site not in sitesIn:
# DIRAC Tier
tier = getSiteTier( site )['Value'][0]
if tier == 0 or tier == '0':
t = 'T0'
elif tier == 1 or tier == '1':
t = 'T1'
elif tier == 3 or tier == '3':
t = 'T3'
else:
t = 'T2'
#Grid Name of the site
gridSiteName = getGOCSiteName( site )
if not gridSiteName['OK']:
raise RSSException, gridSiteName['Message']
gridSiteName = gridSiteName['Value']
#Grid Tier (with a workaround!)
DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
if not DIRACSitesOfGridSites['OK']:
raise RSSException, DIRACSitesOfGridSites['Message']
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len( DIRACSitesOfGridSites ) == 1:
gt = t
else:
gt = self.__getGOCTier( DIRACSitesOfGridSites )
self.rsDB.addOrModifySite( site, t, gridSiteName, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
self.rsDB.addOrModifyGridSite( gridSiteName, gt )
sitesIn.append( site )
def __syncNode(self,
NodeInCS,
resourcesInDB,
resourceType,
serviceType,
site="NULL"):
nodesToUpdate = NodeInCS - resourcesInDB
if len(nodesToUpdate) > 0:
gLogger.debug(str(NodeInCS))
gLogger.debug(str(nodesToUpdate))
# Update Service table
siteInGOCDB = [
self.__getServiceEndpointInfo(node) for node in nodesToUpdate
]
siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
#sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
sites = []
for sInGOCDB in siteInGOCDB:
siteName = getDIRACSiteName(sInGOCDB[0]['SITENAME'])
if not siteName['OK']:
gLogger.error(siteName['Message'])
return siteName
sites.append(siteName['Value'])
sites = Utils.list_sanitize(Utils.list_flatten(sites))
_ = [self.__updateService(s, serviceType) for s in sites]
# Update Resource table
for node in NodeInCS:
if serviceType == "Computing":
resourceType = CS.getCEType(site, node)
if node not in resourcesInDB and node is not None:
try:
siteInGOCDB = self.__getServiceEndpointInfo(
node)[0]['SITENAME']
except IndexError: # No INFO in GOCDB: Node does not exist
gLogger.warn(
"Node %s is not in GOCDB!! Considering that it does not exists!"
% node)
continue
assert (type(siteInGOCDB) == str)
#Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
res = self.rsClient.addOrModifyResource(
node, resourceType, serviceType, site, siteInGOCDB)
if not res['OK']:
gLogger.error(res['Message'])
return res
resourcesInDB.add(node)
def __syncNode(self, NodeInCS, resourcesInDB, resourceType, serviceType, site = "NULL"):
nodesToUpdate = NodeInCS - resourcesInDB
if len(nodesToUpdate) > 0:
gLogger.debug(str(NodeInCS))
gLogger.debug(str(nodesToUpdate))
# Update Service table
siteInGOCDB = [self.__getServiceEndpointInfo(node) for node in nodesToUpdate]
siteInGOCDB = Utils.list_sanitize(siteInGOCDB)
#sites = [Utils.unpack(getDIRACSiteName(s[0]['SITENAME'])) for s in siteInGOCDB]
sites = []
for sInGOCDB in siteInGOCDB:
siteName = getDIRACSiteName( sInGOCDB[ 0 ][ 'SITENAME' ] )
if not siteName[ 'OK' ]:
gLogger.error( siteName[ 'Message' ] )
return siteName
sites.append( siteName[ 'Value' ] )
sites = Utils.list_sanitize( Utils.list_flatten( sites ) )
_ = [ self.__updateService(s, serviceType) for s in sites ]
# Update Resource table
for node in NodeInCS:
if serviceType == "Computing":
resourceType = CS.getCEType(site, node)
if node not in resourcesInDB and node is not None:
try:
siteInGOCDB = self.__getServiceEndpointInfo(node)[0]['SITENAME']
except IndexError: # No INFO in GOCDB: Node does not exist
gLogger.warn("Node %s is not in GOCDB!! Considering that it does not exists!" % node)
continue
assert(type(siteInGOCDB) == str)
#Utils.protect2(self.rsClient.addOrModifyResource, node, resourceType, serviceType, site, siteInGOCDB )
res = self.rsClient.addOrModifyResource( node, resourceType, serviceType, site, siteInGOCDB )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
resourcesInDB.add( node )
def _getInfo( self, requestParams ):
gLogger.info( requestParams )
if not requestParams[ 'name' ]:
gLogger.warn( 'No name given' )
self.finish( { 'success': 'false', 'error': 'We need a Site Name to generate an Overview' } )
elementName = requestParams[ 'name' ][ 0 ]
pub = RPCClient( 'ResourceStatus/Publisher' )
elementStatuses = pub.getElementStatuses( 'Site',
str( elementName ),
None,
'all',
None,
None )
if not elementStatuses[ 'OK' ]:
gLogger.error( elementStatuses[ 'Message' ] )
self.finish( { 'success': 'false', 'error': 'Error getting ElementStatus information' } )
if not elementStatuses[ 'Value' ]:
gLogger.error( 'element "%s" not found' % elementName )
self.finish( { 'success' : 'false', 'error' : 'element "%s" not found' % elementName } )
elementStatus = [ dict( zip( elementStatuses[ 'Columns' ], element ) ) for element in elementStatuses[ 'Value' ] ][ 0 ]
elementStatus[ 'DateEffective' ] = str( elementStatus[ 'DateEffective' ] )
elementStatus[ 'LastCheckTime' ] = str( elementStatus[ 'LastCheckTime' ] )
elementStatus[ 'TokenExpiration' ] = str( elementStatus[ 'TokenExpiration' ] )
gocdb_name = getGOCSiteName( elementName )
if not gocdb_name[ 'OK' ]:
gLogger.error( gocdb_name[ 'Message' ] )
elementStatus[ 'GOCDB' ] = ""
gocdb_name = ''
else:
gocdb_name = gocdb_name[ 'Value' ]
elementStatus[ 'GOCDB' ] = '<a href="https://goc.egi.eu/portal/index.php?Page_Type=Submit_Search&SearchString=%s" target="_blank">%s</a>' % ( gocdb_name, gocdb_name )
dirac_names = getDIRACSiteName( gocdb_name )
if not dirac_names[ 'OK' ]:
gLogger.error( dirac_names[ 'Message' ] )
dirac_names = []
else:
elementStatus[ 'GOCDB' ] += "("
for i in dirac_names['Value']:
elementStatus[ 'GOCDB' ] += "%s " % i
elementStatus[ 'GOCDB' ] += ")"
elementStatus["GGUS"] = '<a href="https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&show_columns_check[]=TICKET_TYPE&show_columns_check[]=AFFECTED_VO&'
elementStatus["GGUS"] += 'show_columns_check[]=AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&'
elementStatus["GGUS"] += 'show_columns_check[]=LAST_UPDATE&show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_hierarchy=all&vo=all&user=&keyword=&involvedsupporter=&assignto=&'
elementStatus["GGUS"] += 'affectedsite=%s&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_provider=&date_type=creation+date&radiotf=1&timeframe=any&from_date=&to_date=&' % gocdb_name
elementStatus["GGUS"] += 'untouched_date=&orderticketsby=GHD_INT_REQUEST_ID&orderhow=descending" target="_blank"> %s tickets</a>' % gocdb_name
convertName = {'CERN-PROD':'CERN',
'INFN-T1':'CNAF',
'FZK-LCG2':'GridKa',
'IN2P3-CC':'IN2P3',
'NIKHEF-ELPROD':'NIKHEF',
'pic':'PIC',
'RAL-LCG2':'RAL',
'SARA-MATRIX':'SARA'}
elog = convertName.get( gocdb_name, "" );
elementStatus['Elog'] = '<a href="https://lblogbook.cern.ch/Operations/?Site=^' + elog + '%24&mode=summary" target="_blank">' + elog + '</a>'
return { 'success' : 'true', 'result' : elementStatus, 'total' : len( elementStatus ) }
def checkUnusedCEs():
global vo, dry, ceBdiiDict
gLogger.notice( 'looking for new computing resources in the BDII database...' )
result = getCEsFromCS()
if not result['OK']:
gLogger.error( 'ERROR: failed to get CEs from CS', result['Message'] )
DIRACExit( -1 )
knownCEs = result['Value']
result = getGridCEs( vo, ceBlackList = knownCEs )
if not result['OK']:
gLogger.error( 'ERROR: failed to get CEs from BDII', result['Message'] )
DIRACExit( -1 )
ceBdiiDict = result['BdiiInfo']
siteDict = result['Value']
if siteDict:
gLogger.notice( 'New resources available:\n' )
for site in siteDict:
diracSite = 'Unknown'
result = getDIRACSiteName( site )
if result['OK']:
diracSite = ','.join( result['Value'] )
ces = siteDict[site].keys()
if ces:
gLogger.notice( " %s, DIRAC site %s" % ( site, diracSite) )
for ce in ces:
gLogger.notice( ' '*4+ce )
gLogger.notice( ' %s, %s' % ( siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System'] ) )
else:
gLogger.notice( 'No new resources available, exiting' )
DIRACExit( 0 )
inp = raw_input( "\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith( 'n' ):
gLogger.notice( 'Nothing else to be done, exiting' )
DIRACExit( 0 )
gLogger.notice( '\nAdding new sites/CEs interactively\n' )
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ''
ces = siteDict[site].keys()
for ce in ces:
country = ce.strip().split('.')[-1].lower()
if len( country ) == 2:
break
if country == 'gov':
country = 'us'
break
if not country or len( country ) != 2:
country = 'xx'
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( '\nThe site %s is not yet in the CS, give it a name' % site )
diracSite = raw_input( '[help|skip|<domain>.<name>.%s]: ' % country )
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice( '%s site details:' % site )
for k,v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice( '%s\t%s' % (k,v) )
gLogger.notice( '\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country )
diracSite = raw_input( '[<domain>.<name>.%s]: ' % country )
try:
domain,siteName,country = diracSite.split('.')
except Exception, x:
gLogger.error( 'ERROR: DIRAC site name does not follow convention: %s' % diracSite )
continue
diracSites = [diracSite]
else:
diracSites = result['Value']
if len( diracSites ) > 1:
gLogger.notice( 'Attention! GOC site %s corresponds to more than one DIRAC sites:' % site )
gLogger.notice( str( diracSites ) )
gLogger.notice( 'Please, pay attention which DIRAC site the new CEs will join\n' )
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]['CEType']
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input( "Add CE %s of type %s to %s? [default yes] [yes|no]: " % ( ce, ceType, diracSite ) )
if yn == '' or yn.lower() == 'y':
newCEs.setdefault( diracSite, [] )
newCEs[diracSite].append( ce )
addedCEs.append( ce )
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % ( diracSite, site, ' '.join( newCEs[diracSite] ) )
gLogger.notice( "\nNew site/CEs will be added with command:\n%s" % cmd )
yn = raw_input( "Add it ? [default yes] [yes|no]: " )
if not ( yn == '' or yn.lower() == 'y' ) :
continue
if dry:
gLogger.notice( "Command is skipped in the dry run" )
else:
result = shellCall( 0, cmd )
if not result['OK']:
gLogger.error( 'Error while executing dirac-admin-add-site command' )
yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " )
if yn == '' or yn.lower().startswith( 'n' ):
if sitesAdded:
gLogger.notice( 'CEs were added at the following sites:' )
for site, diracSite in sitesAdded:
gLogger.notice( "%s\t%s" % ( site, diracSite ) )
DIRACExit( 0 )
else:
exitStatus, stdData, errData = result[ 'Value' ]
if exitStatus:
gLogger.error( 'Error while executing dirac-admin-add-site command\n', '\n'.join( [stdData, errData] ) )
yn = raw_input( "Do you want to continue ? [default no] [yes|no]: " )
if yn == '' or yn.lower().startswith( 'n' ):
if sitesAdded:
gLogger.notice( 'CEs were added at the following sites:' )
for site, diracSite in sitesAdded:
gLogger.notice( "%s\t%s" % ( site, diracSite ) )
DIRACExit( 0 )
else:
sitesAdded.append( ( site, diracSite ) )
gLogger.notice( stdData )
def checkUnusedSEs():
global vo, dry
result = getGridSRMs( vo, unUsed = True )
if not result['OK']:
gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] )
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set( result['Value'] )
else:
csVOs = set( [vo] )
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get( 'GlueServiceVersion', '' )
if not ( version and version.startswith( '2' ) ):
gLogger.debug( 'Skipping SRM service with version %s' % version )
continue
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) )
gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site )
continue
diracSites = result['Value']
yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
if not yn or yn.lower().startswith( 'y' ):
if len( diracSites ) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate( diracSites ):
prompt += '\n[%d] %s' % ( i, s )
prompt += '\nEnter your choice number: '
inp = raw_input( prompt )
try:
ind = int( inp )
except:
gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split( '.' )
recName = '%s-disk' % siteName
inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )
seSection = cfgPath( '/Resources/StorageElements', diracSEName )
changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )
bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
seVOs = csVOs.intersection( bdiiVOs )
changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
accessSection = cfgPath( seSection, 'AccessProtocol.1' )
changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
result = pfnparse( endPoint )
if not result['OK']:
gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
continue
host = result['Value']['Host']
port = result['Value']['Port']
changeSet.add( ( accessSection, 'Host', host ) )
changeSet.add( ( accessSection, 'Port', port ) )
changeSet.add( ( accessSection, 'Access', 'remote' ) )
voPathSection = cfgPath( accessSection, 'VOPath' )
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename( path )
if voFromPath != diracVO:
gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
changeSet.add( ( voPathSection, diracVO, path ) )
path = os.path.dirname( path )
else:
# Try to guess the Path
domain = '.'.join( host.split( '.' )[-2:] )
path = '/dpm/%s/home' % domain
changeSet.add( ( accessSection, 'Path', path ) )
changeSet.add( ( accessSection, 'SpaceToken', '' ) )
changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) )
gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
changeList = list( changeSet )
changeList.sort()
for entry in changeList:
gLogger.notice( entry )
yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )
if not yn or yn.lower().startswith( 'y' ):
changeSetFull = changeSetFull.union( changeSet )
if dry:
if changeSetFull:
gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
changeList = list( changeSetFull )
changeList.sort()
for section, option, value in changeList:
csAPI.setOption( cfgPath( section, option ), value )
yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
if not yn or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
def checkUnusedCEs():
global vo, dry, ceBdiiDict
gLogger.notice("looking for new computing resources in the BDII database...")
result = getCEsFromCS()
if not result["OK"]:
gLogger.error("ERROR: failed to get CEs from CS", result["Message"])
DIRACExit(-1)
knownCEs = result["Value"]
result = getGridCEs(vo, ceBlackList=knownCEs)
if not result["OK"]:
gLogger.error("ERROR: failed to get CEs from BDII", result["Message"])
DIRACExit(-1)
ceBdiiDict = result["BdiiInfo"]
siteDict = result["Value"]
if siteDict:
gLogger.notice("New resources available:\n")
for site in siteDict:
diracSite = "Unknown"
result = getDIRACSiteName(site)
if result["OK"]:
diracSite = ",".join(result["Value"])
ces = siteDict[site].keys()
if ces:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in ces:
gLogger.notice(" " * 4 + ce)
gLogger.notice(
" %s, %s" % (siteDict[site][ce]["CEType"], "%s_%s_%s" % siteDict[site][ce]["System"])
)
else:
gLogger.notice("No new resources available, exiting")
DIRACExit(0)
inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith("n"):
gLogger.notice("Nothing else to be done, exiting")
DIRACExit(0)
gLogger.notice("\nAdding new sites/CEs interactively\n")
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ""
ces = siteDict[site].keys()
for ce in ces:
country = ce.strip().split(".")[-1].lower()
if len(country) == 2:
break
if country == "gov":
country = "us"
break
if not country or len(country) != 2:
country = "xx"
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("\nThe site %s is not yet in the CS, give it a name" % site)
diracSite = raw_input("[help|skip|<domain>.<name>.%s]: " % country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice("%s site details:" % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice("%s\t%s" % (k, v))
gLogger.notice("\nEnter DIRAC site name in the form <domain>.<name>.%s\n" % country)
diracSite = raw_input("[<domain>.<name>.%s]: " % country)
try:
_, _, _ = diracSite.split(".")
except ValueError:
gLogger.error("ERROR: DIRAC site name does not follow convention: %s" % diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result["Value"]
if len(diracSites) > 1:
gLogger.notice("Attention! GOC site %s corresponds to more than one DIRAC sites:" % site)
gLogger.notice(str(diracSites))
gLogger.notice("Please, pay attention which DIRAC site the new CEs will join\n")
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]["CEType"]
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite))
if yn == "" or yn.lower() == "y":
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, " ".join(newCEs[diracSite]))
gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd)
yn = raw_input("Add it ? [default yes] [yes|no]: ")
if not (yn == "" or yn.lower() == "y"):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = shellCall(0, cmd)
if not result["OK"]:
gLogger.error("Error while executing dirac-admin-add-site command")
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == "" or yn.lower().startswith("n"):
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result["Value"]
if exitStatus:
gLogger.error(
"Error while executing dirac-admin-add-site command\n", "\n".join([stdData, errData])
)
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == "" or yn.lower().startswith("n"):
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
if sitesAdded:
gLogger.notice("CEs were added at the following sites:")
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
else:
gLogger.notice("No new CEs were added this time")
csAPI = CSAPI()
if len(args) < 3:
Script.showHelp()
DIRACExit(-1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split(".")
except ValueError:
gLogger.error("The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch")
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result["OK"]:
if result["Value"]:
if len(result["Value"]) > 1:
gLogger.notice("%s GOC site name is associated with several DIRAC sites:" % gridSiteName)
for i, dSite in enumerate(result["Value"]):
gLogger.notice("%d: %s" % (i, dSite))
inp = raw_input("Enter your choice number: ")
try:
inp = int(inp)
except ValueError:
gLogger.error("You should enter an integer number")
DIRACExit(-1)
if 0 <= inp < len(result["Value"]):
diracCSSite = result["Value"][inp]
def doCommand( self, sites = None ):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
if self.client is None:
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
if sites is None:
# from DIRAC.Core.DISET.RPCClient import RPCClient
RPC = RPCClient( "ResourceStatus/ResourceStatus" )
GOC_sites = RPC.getGridSitesList()
if not GOC_sites['OK']:
raise RSSException, where( self, self.doCommand ) + " " + sites['Message']
else:
GOC_sites = GOC_sites['Value']
else:
GOC_sites = [getGOCSiteName( x )['Value'] for x in sites]
try:
res = self.client.getStatus( 'Site', GOC_sites, None, 120 )
except:
gLogger.exception( "Exception when calling GOCDBClient." )
return {}
if not res['OK']:
raise RSSException, where( self, self.doCommand ) + " " + res['Message']
else:
res = res['Value']
if res == None:
return {}
resToReturn = {}
for dt_ID in res:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = res[dt_ID]['SEVERITY']
dt['Description'] = res[dt_ID]['DESCRIPTION'].replace( '\'', '' )
dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName( res[dt_ID]['SITENAME'] )
if not DIRACnames['OK']:
raise RSSException, DIRACnames['Message']
DIRACnames = DIRACnames['Value']
for DIRACname in DIRACnames:
resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
return resToReturn
def _syncResources( self ):
# resources in the DB now
resourcesIn = self.rsDB.getMonitoredsList( 'Resource', paramsList = ['ResourceName'] )
resourcesIn = [r[0] for r in resourcesIn]
# services in the DB now
servicesIn = self.rsDB.getMonitoredsList( 'Service', paramsList = ['ServiceName'] )
servicesIn = [s[0] for s in servicesIn]
# Site-CE mapping in CS now
siteCE = getSiteCEMapping( 'LCG' )['Value']
# Site-SE mapping in CS now
siteSE = getSiteSEMapping( 'LCG' )['Value']
# CEs in CS now
CEList = []
for i in siteCE.values():
for ce in i:
if ce is None:
continue
CEList.append( ce )
# SEs in CS now
SEList = []
for i in siteSE.values():
for x in i:
SEList.append( x )
# SE Nodes in CS now
SENodeList = []
for SE in SEList:
node = getSENodes( SE )['Value'][0]
if node is None:
continue
if node not in SENodeList:
SENodeList.append( node )
# LFC Nodes in CS now
LFCNodeList_L = []
LFCNodeList_C = []
for site in getLFCSites()['Value']:
for readable in ( 'ReadOnly', 'ReadWrite' ):
LFCNode = getLFCNode( site, readable )['Value']
if LFCNode is None or LFCNode == []:
continue
LFCNode = LFCNode[0]
if readable == 'ReadWrite':
if LFCNode not in LFCNodeList_C:
LFCNodeList_C.append( LFCNode )
elif readable == 'ReadOnly':
if LFCNode not in LFCNodeList_L:
LFCNodeList_L.append( LFCNode )
# FTS Nodes in CS now
FTSNodeList = []
sitesWithFTS = getFTSSites()
for site in sitesWithFTS['Value']:
fts = getFTSEndpoint( site )['Value']
if fts is None or fts == []:
continue
fts = fts[0]
if fts not in FTSNodeList:
FTSNodeList.append( fts )
# VOMS Nodes in CS now
VOMSNodeList = getVOMSEndpoints()['Value']
# complete list of resources in CS now
resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList
# list of services in CS now (to be done)
servicesList = []
#remove resources no more in the CS
for res in resourcesIn:
if res not in resourcesList:
self.rsDB.removeResource( res )
self.rsDB.removeStorageElement( resourceName = res )
# add to DB what is in CS now and wasn't before
# CEs
for site in siteCE.keys():
if site == 'LCG.Dummy.ch':
continue
for ce in siteCE[site]:
if ce is None:
continue
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', ce )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( ce )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
serviceType = 'Computing'
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if ce not in resourcesIn:
CEType = getCEType( site, ce )['Value']
ceType = 'CE'
if CEType == 'CREAM':
ceType = 'CREAMCE'
self.rsDB.addOrModifyResource( ce, ceType, serviceType, site, siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( ce )
# SRMs
for srm in SENodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', srm )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( srm )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if srm not in resourcesIn and srm is not None:
self.rsDB.addOrModifyResource( srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( srm )
# LFC_C
for lfc in LFCNodeList_C:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( lfc )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource( lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( lfc )
# LFC_L
for lfc in LFCNodeList_L:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', lfc )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( lfc )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource( lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( lfc )
# FTSs
for fts in FTSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', fts )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( fts )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if fts not in resourcesIn and fts is not None:
self.rsDB.addOrModifyResource( fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( fts )
# VOMSs
for voms in VOMSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', voms )
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex( voms )[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo( 'hostname', trueName )
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName( siteInGOCDB )
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
site = siteInDIRAC['Value']
serviceType = 'VOMS'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append( service )
if service not in servicesIn:
self.rsDB.addOrModifyService( service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
servicesIn.append( service )
if voms not in resourcesIn and voms is not None:
self.rsDB.addOrModifyResource( voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active', 'init',
datetime.datetime.utcnow().replace( microsecond = 0 ), 'RS_SVC',
datetime.datetime( 9999, 12, 31, 23, 59, 59 ) )
resourcesIn.append( voms )
#remove services no more in the CS
for ser in servicesIn:
if ser not in servicesList:
serType = ser.split( '@' )[0]
if serType != 'VO-BOX':
self.rsDB.removeService( ser )
self.rsDB.removeResource( serviceName = ser )
site = ser.split( '@' )[1]
if serType == 'Storage':
self.rsDB.removeStorageElement( siteName = site )
def doCommand(self, sites=None):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
self.APIs = initAPIs(self.__APIs__, self.APIs)
try:
if sites is None:
GOC_sites = self.APIs['ResourceStatusClient'].getGridSite(
meta={'columns': 'GridSiteName'})
if not GOC_sites['OK']:
return {'Result': GOC_sites}
GOC_sites = [gs[0] for gs in GOC_sites['Value']]
else:
GOC_sites = [getGOCSiteName(x)['Value'] for x in sites]
resGOC = self.APIs['GOCDBClient'].getStatus(
'Site', GOC_sites, None, 120)
if not resGOC['OK']:
return {'Result': resGOC}
resGOC = resGOC['Value']
if resGOC == None:
resGOC = []
res = {}
for dt_ID in resGOC:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = resGOC[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = resGOC[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = resGOC[dt_ID]['SEVERITY']
dt['Description'] = resGOC[dt_ID]['DESCRIPTION'].replace(
'\'', '')
dt['Link'] = resGOC[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME'])
if not DIRACnames['OK']:
return {'Result': DIRACnames}
for DIRACname in DIRACnames['Value']:
res[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
res = S_OK(res)
except Exception, e:
_msg = '%s (%s): %s' % (self.__class__.__name__, self.args, e)
gLogger.exception(_msg)
return {'Result': S_ERROR(_msg)}
def _syncResources(self):
# resources in the DB now
resourcesIn = self.rsDB.getMonitoredsList('Resource',
paramsList=['ResourceName'])
resourcesIn = [r[0] for r in resourcesIn]
# services in the DB now
servicesIn = self.rsDB.getMonitoredsList('Service',
paramsList=['ServiceName'])
servicesIn = [s[0] for s in servicesIn]
# Site-CE mapping in CS now
siteCE = getSiteCEMapping('LCG')['Value']
# Site-SE mapping in CS now
siteSE = getSiteSEMapping('LCG')['Value']
# CEs in CS now
CEList = []
for i in siteCE.values():
for ce in i:
if ce is None:
continue
CEList.append(ce)
# SEs in CS now
SEList = []
for i in siteSE.values():
for x in i:
SEList.append(x)
# SE Nodes in CS now
SENodeList = []
for SE in SEList:
node = getSENodes(SE)['Value'][0]
if node is None:
continue
if node not in SENodeList:
SENodeList.append(node)
# LFC Nodes in CS now
LFCNodeList_L = []
LFCNodeList_C = []
for site in getLFCSites()['Value']:
for readable in ('ReadOnly', 'ReadWrite'):
LFCNode = getLFCNode(site, readable)['Value']
if LFCNode is None or LFCNode == []:
continue
LFCNode = LFCNode[0]
if readable == 'ReadWrite':
if LFCNode not in LFCNodeList_C:
LFCNodeList_C.append(LFCNode)
elif readable == 'ReadOnly':
if LFCNode not in LFCNodeList_L:
LFCNodeList_L.append(LFCNode)
# FTS Nodes in CS now
FTSNodeList = []
sitesWithFTS = getFTSSites()
for site in sitesWithFTS['Value']:
fts = getFTSEndpoint(site)['Value']
if fts is None or fts == []:
continue
fts = fts[0]
if fts not in FTSNodeList:
FTSNodeList.append(fts)
# VOMS Nodes in CS now
VOMSNodeList = getVOMSEndpoints()['Value']
# complete list of resources in CS now
resourcesList = CEList + SENodeList + LFCNodeList_L + LFCNodeList_C + FTSNodeList + VOMSNodeList
# list of services in CS now (to be done)
servicesList = []
#remove resources no more in the CS
for res in resourcesIn:
if res not in resourcesList:
self.rsDB.removeResource(res)
self.rsDB.removeStorageElement(resourceName=res)
# add to DB what is in CS now and wasn't before
# CEs
for site in siteCE.keys():
if site == 'LCG.Dummy.ch':
continue
for ce in siteCE[site]:
if ce is None:
continue
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', ce)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(ce)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
serviceType = 'Computing'
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if ce not in resourcesIn:
CEType = getCEType(site, ce)['Value']
ceType = 'CE'
if CEType == 'CREAM':
ceType = 'CREAMCE'
self.rsDB.addOrModifyResource(
ce, ceType, serviceType, site, siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(ce)
# SRMs
for srm in SENodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', srm)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(srm)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if srm not in resourcesIn and srm is not None:
self.rsDB.addOrModifyResource(
srm, 'SE', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(srm)
# LFC_C
for lfc in LFCNodeList_C:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', lfc)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(lfc)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource(
lfc, 'LFC_C', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(lfc)
# LFC_L
for lfc in LFCNodeList_L:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', lfc)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(lfc)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if lfc not in resourcesIn and lfc is not None:
self.rsDB.addOrModifyResource(
lfc, 'LFC_L', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(lfc)
# FTSs
for fts in FTSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', fts)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(fts)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
sites = siteInDIRAC['Value']
serviceType = 'Storage'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if fts not in resourcesIn and fts is not None:
self.rsDB.addOrModifyResource(
fts, 'FTS', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(fts)
# VOMSs
for voms in VOMSNodeList:
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', voms)
if not siteInGOCDB['OK']:
raise RSSException, siteInGOCDB['Message']
if siteInGOCDB['Value'] == []:
trueName = socket.gethostbyname_ex(voms)[0]
siteInGOCDB = self.GOCDBClient.getServiceEndpointInfo(
'hostname', trueName)
try:
siteInGOCDB = siteInGOCDB['Value'][0]['SITENAME']
except IndexError:
continue
siteInDIRAC = getDIRACSiteName(siteInGOCDB)
if not siteInDIRAC['OK']:
raise RSSException, siteInDIRAC['Message']
site = siteInDIRAC['Value']
serviceType = 'VOMS'
for site in sites:
service = serviceType + '@' + site
if service not in servicesList:
servicesList.append(service)
if service not in servicesIn:
self.rsDB.addOrModifyService(
service, serviceType, site, 'Active', 'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
servicesIn.append(service)
if voms not in resourcesIn and voms is not None:
self.rsDB.addOrModifyResource(
voms, 'VOMS', serviceType, 'NULL', siteInGOCDB, 'Active',
'init',
datetime.datetime.utcnow().replace(microsecond=0),
'RS_SVC', datetime.datetime(9999, 12, 31, 23, 59, 59))
resourcesIn.append(voms)
#remove services no more in the CS
for ser in servicesIn:
if ser not in servicesList:
serType = ser.split('@')[0]
if serType != 'VO-BOX':
self.rsDB.removeService(ser)
self.rsDB.removeResource(serviceName=ser)
site = ser.split('@')[1]
if serType == 'Storage':
self.rsDB.removeStorageElement(siteName=site)
def _getInfo( self, requestParams ):
gLogger.info( requestParams )
if not requestParams[ 'name' ]:
gLogger.warn( 'No name given' )
self.finish( { 'success': 'false', 'error': 'We need a Site Name to generate an Overview' } )
elementName = requestParams[ 'name' ][ 0 ]
pub = RPCClient( 'ResourceStatus/Publisher' )
elementStatuses = pub.getElementStatuses( 'Site',
str( elementName ),
None,
'all',
None,
None )
if not elementStatuses[ 'OK' ]:
gLogger.error( elementStatuses[ 'Message' ] )
self.finish( { 'success': 'false', 'error': 'Error getting ElementStatus information' } )
if not elementStatuses[ 'Value' ]:
gLogger.error( 'element "%s" not found' % elementName )
self.finish( { 'success' : 'false', 'error' : 'element "%s" not found' % elementName } )
elementStatus = [ dict( zip( elementStatuses[ 'Columns' ], element ) ) for element in elementStatuses[ 'Value' ] ][ 0 ]
elementStatus[ 'DateEffective' ] = str( elementStatus[ 'DateEffective' ] )
elementStatus[ 'LastCheckTime' ] = str( elementStatus[ 'LastCheckTime' ] )
elementStatus[ 'TokenExpiration' ] = str( elementStatus[ 'TokenExpiration' ] )
gocdb_name = getGOCSiteName( elementName )
if not gocdb_name[ 'OK' ]:
gLogger.error( gocdb_name[ 'Message' ] )
elementStatus[ 'GOCDB' ] = ""
gocdb_name = ''
else:
gocdb_name = gocdb_name[ 'Value' ]
elementStatus[ 'GOCDB' ] = '<a href="https://goc.egi.eu/portal/index.php?Page_Type=Submit_Search&SearchString=%s" target="_blank">%s</a>' % ( gocdb_name, gocdb_name )
dirac_names = getDIRACSiteName( gocdb_name )
if not dirac_names[ 'OK' ]:
gLogger.error( dirac_names[ 'Message' ] )
dirac_names = []
else:
elementStatus[ 'GOCDB' ] += "("
for i in dirac_names['Value']:
elementStatus[ 'GOCDB' ] += "%s " % i
elementStatus[ 'GOCDB' ] += ")"
elementStatus["GGUS"] = '<a href="https://ggus.eu/ws/ticket_search.php?show_columns_check[]=REQUEST_ID&show_columns_check[]=TICKET_TYPE&show_columns_check[]=AFFECTED_VO&'
elementStatus["GGUS"] += 'show_columns_check[]=AFFECTED_SITE&show_columns_check[]=PRIORITY&show_columns_check[]=RESPONSIBLE_UNIT&show_columns_check[]=STATUS&show_columns_check[]=DATE_OF_CREATION&'
elementStatus["GGUS"] += 'show_columns_check[]=LAST_UPDATE&show_columns_check[]=TYPE_OF_PROBLEM&show_columns_check[]=SUBJECT&ticket=&supportunit=all&su_hierarchy=all&vo=all&user=&keyword=&involvedsupporter=&assignto=&'
elementStatus["GGUS"] += 'affectedsite=%s&specattrib=0&status=open&priority=all&typeofproblem=all&ticketcategory=&mouarea=&technology_provider=&date_type=creation+date&radiotf=1&timeframe=any&from_date=&to_date=&' % gocdb_name
elementStatus["GGUS"] += 'untouched_date=&orderticketsby=GHD_INT_REQUEST_ID&orderhow=descending" target="_blank"> %s tickets</a>' % gocdb_name
convertName = {'CERN-PROD':'CERN',
'INFN-T1':'CNAF',
'FZK-LCG2':'GridKa',
'IN2P3-CC':'IN2P3',
'NIKHEF-ELPROD':'NIKHEF',
'pic':'PIC',
'RAL-LCG2':'RAL',
'SARA-MATRIX':'SARA'}
elog = convertName.get( gocdb_name, "" );
elementStatus['Elog'] = '<a href="https://lblogbook.cern.ch/Operations/?Site=^' + elog + '%24&mode=summary" target="_blank">' + elog + '</a>'
return { 'success' : 'true', 'result' : elementStatus, 'total' : len( elementStatus ) }
def getSiteUpdates(vo, bdiiInfo=None, log=None):
""" Get all the necessary updates for the already defined sites and CEs
"""
def addToChangeSet(entry, changeSet):
""" Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo)
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath('/Resources', 'Sites', siteName.split('.')[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get('Coordinates', 'Unknown')
mail = siteDict.get('Mail', 'Unknown').replace(' ', '')
description = siteDict.get('Description', 'Unknown')
description = description.replace(' ,', ',')
longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip()
latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact', '').replace('mailto:', '').strip()
newdescription = ceBdiiDict[site].get('GlueSiteDescription', '').strip()
newdescription = ", ".join([line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, 'Coordinates', coor, newcoor), changeSet)
addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet)
addToChangeSet((siteSection, 'Description', description, newdescription), changeSet)
ces = gConfig.getValue(cfgPath(siteSection, 'CE'), [])
for ce in ces:
ceSection = cfgPath(siteSection, 'CEs', ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get(ce, None):
log.notice("Adding new CE", "%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]['CEs'].get(ce, None)
if ceInfo is None:
ceType = ceDict.get('CEType', '')
continue
# Current CS CE info
arch = ceDict.get('architecture', 'Unknown')
OS = ceDict.get('OS', 'Unknown')
si00 = ceDict.get('SI00', 'Unknown')
ceType = ceDict.get('CEType', 'Unknown')
ram = ceDict.get('MaxRAM', 'Unknown')
submissionMode = ceDict.get('SubmissionMode', 'Unknown')
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get('GlueHostArchitecturePlatformType', '').strip()
systemName = ceInfo.get('GlueHostOperatingSystemName', '').strip()
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion', '').strip()
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease', '').strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join((systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get('GlueCEImplementationName', '').strip()
if newCEType:
break
if newCEType == 'ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC', 'CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ''
# Adding CE data to the change list
addToChangeSet((ceSection, 'architecture', arch, newarch), changeSet)
addToChangeSet((ceSection, 'OS', OS, newOS), changeSet)
addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet)
addToChangeSet((ceSection, 'CEType', ceType, newCEType), changeSet)
addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, 'SubmissionMode', submissionMode, newSubmissionMode), changeSet)
queues = ceInfo['Queues'].keys()
for queue in queues:
queueInfo = ceInfo['Queues'][queue]
queueStatus = queueInfo['GlueCEStateStatus']
queueSection = cfgPath(ceSection, 'Queues', queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result['OK']:
queueDict = result['Value']
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue", "%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get('maxCPUTime', 'Unknown')
si00 = queueDict.get('SI00', 'Unknown')
maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown')
# Current BDII queue info
newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '')
if newMaxCPUTime == "4" * len(newMaxCPUTime) or newMaxCPUTime == "9" * len(newMaxCPUTime):
newMaxCPUTime = ''
wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime', '')
if wallTime == "4" * len(wallTime) or wallTime == "9" * len(wallTime):
wallTime = ''
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ''
caps = queueInfo.get('GlueCECapability', [])
if isinstance(caps, basestring):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split('=')[-1]
# Adding queue info to the CS
addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime, newMaxCPUTime), changeSet)
addToChangeSet((queueSection, 'SI00', si00, newSI00), changeSet)
if maxTotalJobs == "Unknown":
newTotalJobs = min(1000, int(int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) / 2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet((queueSection, 'MaxTotalJobs', '', newTotalJobs), changeSet)
addToChangeSet((queueSection, 'MaxWaitingJobs', '', newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get('VO', ''):
VOs = set([q.strip() for q in queueDict.get('VO', '').split(',') if q])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ','.join(VOs)
addToChangeSet((queueSection, 'VO', '', newVOs), changeSet)
return S_OK(changeSet)
if len(args) < 3:
Script.showHelp()
DIRACExit(-1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split('.')
except ValueError:
gLogger.error(
"The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch"
)
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
newSite = True
if result['OK']:
if result['Value']:
if len(result['Value']) > 1:
gLogger.notice(
'%s GOC site name is associated with several DIRAC sites:'
% gridSiteName)
for i, dSite in enumerate(result['Value']):
gLogger.notice('%d: %s' % (i, dSite))
inp = raw_input('Enter your choice number: ')
try:
inp = int(inp)
except ValueError:
gLogger.error('You should enter an integer number')
DIRACExit(-1)
def doCommand( self, sites = None ):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
self.APIs = initAPIs( self.__APIs__, self.APIs )
try:
if sites is None:
GOC_sites = self.APIs[ 'ResourceStatusClient' ].getGridSite( meta = { 'columns' : 'GridSiteName' })
if not GOC_sites['OK']:
return { 'Result' : GOC_sites }
GOC_sites = [ gs[0] for gs in GOC_sites['Value'] ]
else:
GOC_sites = [ getGOCSiteName( x )['Value'] for x in sites ]
resGOC = self.APIs[ 'GOCDBClient' ].getStatus( 'Site', GOC_sites, None, 120 )
if not resGOC['OK']:
return { 'Result' : resGOC }
resGOC = resGOC['Value']
if resGOC == None:
resGOC = []
res = {}
for dt_ID in resGOC:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = resGOC[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = resGOC[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = resGOC[dt_ID]['SEVERITY']
dt['Description'] = resGOC[dt_ID]['DESCRIPTION'].replace( '\'', '' )
dt['Link'] = resGOC[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName( res[dt_ID]['SITENAME'] )
if not DIRACnames['OK']:
return { 'Result' : DIRACnames }
for DIRACname in DIRACnames['Value']:
res[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
res = S_OK( res )
except Exception, e:
_msg = '%s (%s): %s' % ( self.__class__.__name__, self.args, e )
gLogger.exception( _msg )
return { 'Result' : S_ERROR( _msg ) }
def doCommand(self, sites=None):
"""
Returns downtimes information for all the sites in input.
:params:
:attr:`sites`: list of site names (when not given, take every site)
:returns:
{'SiteName': {'SEVERITY': 'OUTAGE'|'AT_RISK',
'StartDate': 'aDate', ...} ... }
"""
if self.client is None:
from DIRAC.Core.LCG.GOCDBClient import GOCDBClient
self.client = GOCDBClient()
if sites is None:
# from DIRAC.Core.DISET.RPCClient import RPCClient
RPC = RPCClient("ResourceStatus/ResourceStatus")
GOC_sites = RPC.getGridSitesList()
if not GOC_sites['OK']:
raise RSSException, where(
self, self.doCommand) + " " + sites['Message']
else:
GOC_sites = GOC_sites['Value']
else:
GOC_sites = [getGOCSiteName(x)['Value'] for x in sites]
try:
res = self.client.getStatus('Site', GOC_sites, None, 120)
except:
gLogger.exception("Exception when calling GOCDBClient.")
return {}
if not res['OK']:
raise RSSException, where(self,
self.doCommand) + " " + res['Message']
else:
res = res['Value']
if res == None:
return {}
resToReturn = {}
for dt_ID in res:
try:
dt = {}
dt['ID'] = dt_ID
dt['StartDate'] = res[dt_ID]['FORMATED_START_DATE']
dt['EndDate'] = res[dt_ID]['FORMATED_END_DATE']
dt['Severity'] = res[dt_ID]['SEVERITY']
dt['Description'] = res[dt_ID]['DESCRIPTION'].replace('\'', '')
dt['Link'] = res[dt_ID]['GOCDB_PORTAL_URL']
DIRACnames = getDIRACSiteName(res[dt_ID]['SITENAME'])
if not DIRACnames['OK']:
raise RSSException, DIRACnames['Message']
DIRACnames = DIRACnames['Value']
for DIRACname in DIRACnames:
resToReturn[dt_ID.split()[0] + ' ' + DIRACname] = dt
except KeyError:
continue
return resToReturn
def _syncSites( self ):
"""
Sync DB content with sites that are in the CS
"""
def getGOCTier(sitesList):
return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))
# sites in the DB now
#sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))
sites = self.rsClient.getSite()
if not sites[ 'OK' ]:
gLogger.error( sites[ 'Message' ] )
return sites
sitesDB = set( [ site[0] for site in sites[ 'Value' ] ] )
# sites in CS now
sitesCS = set( CS.getSites() )
gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" % (len(sitesCS), len(sitesDB)))
# remove sites and associated resources, services, and storage
# elements from the DB that are not in the CS:
for s in sitesDB - sitesCS:
gLogger.info("Purging Site %s (not in CS anymore)" % s)
self.__purge_site(s)
# add to DB what is missing
gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
for site in sitesCS - sitesDB:
siteType = site.split(".")[0]
# DIRAC Tier
tier = "T" + str(CS.getSiteTier( site ))
if siteType == "LCG":
# Grid Name of the site
#gridSiteName = Utils.unpack(getGOCSiteName(site))
gridSiteName = getGOCSiteName( site )
if not gridSiteName[ 'OK' ]:
gLogger.error( gridSiteName[ 'Message' ] )
return gridSiteName
gridSiteName = gridSiteName[ 'Value' ]
# Grid Tier (with a workaround!)
#DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
DIRACSitesOfGridSites = getDIRACSiteName( gridSiteName )
if not DIRACSitesOfGridSites[ 'OK' ]:
gLogger.error( DIRACSitesOfGridSites[ 'Message' ] )
return DIRACSitesOfGridSites
DIRACSitesOfGridSites = DIRACSitesOfGridSites[ 'Value' ]
if len( DIRACSitesOfGridSites ) == 1:
gt = tier
else:
gt = getGOCTier( DIRACSitesOfGridSites )
#Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
res = self.rsClient.addOrModifyGridSite( gridSiteName, gt )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
res = self.rsClient.addOrModifySite( site, tier, gridSiteName )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
elif siteType == "DIRAC":
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
res = self.rsClient.addOrModifySite( site, tier, "NULL" )
if not res[ 'OK' ]:
gLogger.error( res[ 'Message' ] )
return res
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result['OK']:
gLogger.error('Failed to look up SRMs in BDII', result['Message'])
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set(result['Value'])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get('GlueServiceVersion', '')
if not (version and version.startswith('2')):
gLogger.debug('Skipping SRM service with version %s' % version)
continue
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('Unused se %s is detected at unused site %s' % (gridSE, site))
gLogger.notice('Consider adding site %s to the DIRAC CS' % site)
continue
diracSites = result['Value']
yn = raw_input(
'\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' %
(gridSE, str(diracSites)))
if not yn or yn.lower().startswith('y'):
if len(diracSites) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate(diracSites):
prompt += '\n[%d] %s' % (i, s)
prompt += '\nEnter your choice number: '
inp = raw_input(prompt)
try:
ind = int(inp)
except BaseException:
gLogger.notice('Can not interpret your choice: %s, try again later' % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split('.')
recName = '%s-disk' % siteName
inp = raw_input('Give a DIRAC name to the grid SE %s, default %s : ' % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice('Adding new SE %s at site %s' % (diracSEName, diracSite))
seSection = cfgPath('/Resources/StorageElements', diracSEName)
changeSet.add((seSection, 'BackendType', seDict.get('GlueSEImplementationName', 'Unknown')))
changeSet.add((seSection, 'Description', seDict.get('GlueSEName', 'Unknown')))
bdiiVOs = set([re.sub('^VO:', '', rule) for rule in srmDict.get('GlueServiceAccessControlBaseRule', [])])
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, 'VO', ','.join(seVOs)))
accessSection = cfgPath(seSection, 'AccessProtocol.1')
changeSet.add((accessSection, 'Protocol', 'srm'))
changeSet.add((accessSection, 'PluginName', 'SRM2'))
endPoint = srmDict.get('GlueServiceEndpoint', '')
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, 'Host', host))
changeSet.add((accessSection, 'Port', port))
changeSet.add((accessSection, 'Access', 'remote'))
voPathSection = cfgPath(accessSection, 'VOPath')
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice('\n!!! Warning: non-conventional VO path: %s\n' % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = '.'.join(host.split('.')[-2:])
path = '/dpm/%s/home' % domain
changeSet.add((accessSection, 'Path', path))
changeSet.add((accessSection, 'SpaceToken', ''))
changeSet.add((accessSection, 'WSUrl', '/srm/managerv2?SFN='))
gLogger.notice('SE %s will be added with the following parameters' % diracSEName)
changeList = sorted(changeSet)
for entry in changeList:
gLogger.notice(entry)
yn = raw_input('Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName)
if not yn or yn.lower().startswith('y'):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice('Skipping commit of the new SE data in a dry run')
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error('Failed to initialize CSAPI object', result['Message'])
DIRACExit(-1)
changeList = sorted(changeSetFull)
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input('New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ')
if not yn or yn.lower().startswith('y'):
result = csAPI.commit()
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
def _syncSites(self):
"""
Sync DB content with sites that are in the CS
"""
def getGOCTier(sitesList):
return "T" + str(min([int(v) for v in CS.getSiteTiers(sitesList)]))
# sites in the DB now
#sitesDB = set((s[0] for s in Utils.unpack(self.rsClient.getSite())))
sites = self.rsClient.getSite()
if not sites['OK']:
gLogger.error(sites['Message'])
return sites
sitesDB = set([site[0] for site in sites['Value']])
# sites in CS now
sitesCS = set(CS.getSites())
gLogger.info("Syncing Sites from CS: %d sites in CS, %d sites in DB" %
(len(sitesCS), len(sitesDB)))
# remove sites and associated resources, services, and storage
# elements from the DB that are not in the CS:
for s in sitesDB - sitesCS:
gLogger.info("Purging Site %s (not in CS anymore)" % s)
self.__purge_site(s)
# add to DB what is missing
gLogger.info("Updating %d Sites in DB" % len(sitesCS - sitesDB))
for site in sitesCS - sitesDB:
siteType = site.split(".")[0]
# DIRAC Tier
tier = "T" + str(CS.getSiteTier(site))
if siteType == "LCG":
# Grid Name of the site
#gridSiteName = Utils.unpack(getGOCSiteName(site))
gridSiteName = getGOCSiteName(site)
if not gridSiteName['OK']:
gLogger.error(gridSiteName['Message'])
return gridSiteName
gridSiteName = gridSiteName['Value']
# Grid Tier (with a workaround!)
#DIRACSitesOfGridSites = Utils.unpack(getDIRACSiteName(gridSiteName))
DIRACSitesOfGridSites = getDIRACSiteName(gridSiteName)
if not DIRACSitesOfGridSites['OK']:
gLogger.error(DIRACSitesOfGridSites['Message'])
return DIRACSitesOfGridSites
DIRACSitesOfGridSites = DIRACSitesOfGridSites['Value']
if len(DIRACSitesOfGridSites) == 1:
gt = tier
else:
gt = getGOCTier(DIRACSitesOfGridSites)
#Utils.protect2(self.rsClient.addOrModifyGridSite, gridSiteName, gt)
res = self.rsClient.addOrModifyGridSite(gridSiteName, gt)
if not res['OK']:
gLogger.error(res['Message'])
return res
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, gridSiteName )
res = self.rsClient.addOrModifySite(site, tier, gridSiteName)
if not res['OK']:
gLogger.error(res['Message'])
return res
elif siteType == "DIRAC":
#Utils.protect2(self.rsClient.addOrModifySite, site, tier, "NULL" )
res = self.rsClient.addOrModifySite(site, tier, "NULL")
if not res['OK']:
gLogger.error(res['Message'])
return res
def checkUnusedCEs():
global vo, dry, ceBdiiDict, hostURL, glue2
gLogger.notice('looking for new computing resources in the BDII database...')
result = getCEsFromCS()
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from CS', result['Message'])
DIRACExit(-1)
knownCEs = result['Value']
result = getGridCEs(vo, ceBlackList=knownCEs, hostURL=hostURL, glue2=glue2)
if not result['OK']:
gLogger.error('ERROR: failed to get CEs from BDII', result['Message'])
DIRACExit(-1)
ceBdiiDict = result['BdiiInfo']
siteDict = result['Value']
if siteDict:
gLogger.notice('New resources available:\n')
for site in siteDict:
diracSite = 'Unknown'
result = getDIRACSiteName(site)
if result['OK']:
diracSite = ','.join(result['Value'])
ces = siteDict[site].keys() # pylint: disable=no-member
if ces:
gLogger.notice(" %s, DIRAC site %s" % (site, diracSite))
for ce in ces:
gLogger.notice(' ' * 4 + ce)
gLogger.notice(' %s, %s' % (siteDict[site][ce]['CEType'], '%s_%s_%s' % siteDict[site][ce]['System']))
else:
gLogger.notice('No new resources available, exiting')
DIRACExit(0)
inp = raw_input("\nDo you want to add sites ? [default=yes] [yes|no]: ")
inp = inp.strip()
if not inp and inp.lower().startswith('n'):
gLogger.notice('Nothing else to be done, exiting')
DIRACExit(0)
gLogger.notice('\nAdding new sites/CEs interactively\n')
sitesAdded = []
for site in siteDict:
# Get the country code:
country = ''
ces = siteDict[site].keys() # pylint: disable=no-member
for ce in ces:
country = ce.strip().split('.')[-1].lower()
if len(country) == 2:
break
if country == 'gov':
country = 'us'
break
if not country or len(country) != 2:
country = 'xx'
result = getDIRACSiteName(site)
if not result['OK']:
gLogger.notice('\nThe site %s is not yet in the CS, give it a name' % site)
diracSite = raw_input('[help|skip|<domain>.<name>.%s]: ' % country)
if diracSite.lower() == "skip":
continue
if diracSite.lower() == "help":
gLogger.notice('%s site details:' % site)
for k, v in ceBdiiDict[site].items():
if k != "CEs":
gLogger.notice('%s\t%s' % (k, v))
gLogger.notice('\nEnter DIRAC site name in the form <domain>.<name>.%s\n' % country)
diracSite = raw_input('[<domain>.<name>.%s]: ' % country)
try:
_, _, _ = diracSite.split('.')
except ValueError:
gLogger.error('ERROR: DIRAC site name does not follow convention: %s' % diracSite)
continue
diracSites = [diracSite]
else:
diracSites = result['Value']
if len(diracSites) > 1:
gLogger.notice('Attention! GOC site %s corresponds to more than one DIRAC sites:' % site)
gLogger.notice(str(diracSites))
gLogger.notice('Please, pay attention which DIRAC site the new CEs will join\n')
newCEs = {}
addedCEs = []
for ce in ces:
ceType = siteDict[site][ce]['CEType']
for diracSite in diracSites:
if ce in addedCEs:
continue
yn = raw_input("Add CE %s of type %s to %s? [default yes] [yes|no]: " % (ce, ceType, diracSite))
if yn == '' or yn.lower() == 'y':
newCEs.setdefault(diracSite, [])
newCEs[diracSite].append(ce)
addedCEs.append(ce)
for diracSite in diracSites:
if diracSite in newCEs:
cmd = "dirac-admin-add-site %s %s %s" % (diracSite, site, ' '.join(newCEs[diracSite]))
gLogger.notice("\nNew site/CEs will be added with command:\n%s" % cmd)
yn = raw_input("Add it ? [default yes] [yes|no]: ")
if not (yn == '' or yn.lower() == 'y'):
continue
if dry:
gLogger.notice("Command is skipped in the dry run")
else:
result = systemCall(0, shlex.split(cmd))
if not result['OK']:
gLogger.error('Error while executing dirac-admin-add-site command')
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
exitStatus, stdData, errData = result['Value']
if exitStatus:
gLogger.error('Error while executing dirac-admin-add-site command\n', '\n'.join([stdData, errData]))
yn = raw_input("Do you want to continue ? [default no] [yes|no]: ")
if yn == '' or yn.lower().startswith('n'):
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
DIRACExit(0)
else:
sitesAdded.append((site, diracSite))
gLogger.notice(stdData)
if sitesAdded:
gLogger.notice('CEs were added at the following sites:')
for site, diracSite in sitesAdded:
gLogger.notice("%s\t%s" % (site, diracSite))
else:
gLogger.notice('No new CEs were added this time')
def getSiteUpdates(vo, bdiiInfo=None, log=None):
""" Get all the necessary updates for the already defined sites and CEs
"""
def addToChangeSet(entry, changeSet):
""" Inner function to update changeSet with entry (a tuple)
:param tuple entry: entry to add to changeSet
:param set changeSet: set collecting stuff to change
"""
_section, _option, value, new_value = entry
if new_value and new_value != value:
changeSet.add(entry)
if log is None:
log = gLogger
ceBdiiDict = bdiiInfo
if bdiiInfo is None:
result = getBdiiCEInfo(vo)
if not result['OK']:
return result
ceBdiiDict = result['Value']
changeSet = set()
for site in ceBdiiDict:
result = getDIRACSiteName(site)
if not result['OK']:
continue
siteNames = result['Value']
for siteName in siteNames:
siteSection = cfgPath('/Resources', 'Sites',
siteName.split('.')[0], siteName)
result = gConfig.getOptionsDict(siteSection)
if not result['OK']:
continue
siteDict = result['Value']
# Current CS values
coor = siteDict.get('Coordinates', 'Unknown')
mail = siteDict.get('Mail', 'Unknown').replace(' ', '')
description = siteDict.get('Description', 'Unknown')
description = description.replace(' ,', ',')
longitude = ceBdiiDict[site].get('GlueSiteLongitude', '').strip()
latitude = ceBdiiDict[site].get('GlueSiteLatitude', '').strip()
# Current BDII value
newcoor = ''
if longitude and latitude:
newcoor = "%s:%s" % (longitude, latitude)
newmail = ceBdiiDict[site].get('GlueSiteSysAdminContact',
'').replace('mailto:', '').strip()
newdescription = ceBdiiDict[site].get('GlueSiteDescription',
'').strip()
newdescription = ", ".join(
[line.strip() for line in newdescription.split(",")])
# Adding site data to the changes list
addToChangeSet((siteSection, 'Coordinates', coor, newcoor),
changeSet)
addToChangeSet((siteSection, 'Mail', mail, newmail), changeSet)
addToChangeSet(
(siteSection, 'Description', description, newdescription),
changeSet)
ces = gConfig.getValue(cfgPath(siteSection, 'CE'), [])
for ce in ces:
ceSection = cfgPath(siteSection, 'CEs', ce)
ceDict = {}
result = gConfig.getOptionsDict(ceSection)
if result['OK']:
ceDict = result['Value']
else:
if ceBdiiDict[site]['CEs'].get(ce, None):
log.notice("Adding new CE",
"%s to site %s/%s" % (ce, siteName, site))
ceInfo = ceBdiiDict[site]['CEs'].get(ce, None)
if ceInfo is None:
ceType = ceDict.get('CEType', '')
continue
# Current CS CE info
arch = ceDict.get('architecture', 'Unknown')
OS = ceDict.get('OS', 'Unknown')
si00 = ceDict.get('SI00', 'Unknown')
ceType = ceDict.get('CEType', 'Unknown')
ram = ceDict.get('MaxRAM', 'Unknown')
submissionMode = ceDict.get('SubmissionMode', 'Unknown')
# Current BDII CE info
newarch = ceBdiiDict[site]['CEs'][ce].get(
'GlueHostArchitecturePlatformType', '').strip()
systemName = ceInfo.get('GlueHostOperatingSystemName',
'').strip()
systemVersion = ceInfo.get('GlueHostOperatingSystemVersion',
'').strip()
systemRelease = ceInfo.get('GlueHostOperatingSystemRelease',
'').strip()
newOS = ''
if systemName and systemVersion and systemRelease:
newOS = '_'.join(
(systemName, systemVersion, systemRelease))
newsi00 = ceInfo.get('GlueHostBenchmarkSI00', '').strip()
newCEType = 'Unknown'
for queue in ceInfo['Queues']:
queueDict = ceInfo['Queues'][queue]
newCEType = queueDict.get('GlueCEImplementationName',
'').strip()
if newCEType:
break
if newCEType == 'ARC-CE':
newCEType = 'ARC'
newSubmissionMode = None
if newCEType in ['ARC', 'CREAM']:
newSubmissionMode = "Direct"
newRAM = ceInfo.get('GlueHostMainMemoryRAMSize', '').strip()
# Protect from unreasonable values
if newRAM and int(newRAM) > 150000:
newRAM = ''
# Adding CE data to the change list
addToChangeSet((ceSection, 'architecture', arch, newarch),
changeSet)
addToChangeSet((ceSection, 'OS', OS, newOS), changeSet)
addToChangeSet((ceSection, 'SI00', si00, newsi00), changeSet)
addToChangeSet((ceSection, 'CEType', ceType, newCEType),
changeSet)
addToChangeSet((ceSection, 'MaxRAM', ram, newRAM), changeSet)
if submissionMode == "Unknown" and newSubmissionMode:
addToChangeSet((ceSection, 'SubmissionMode',
submissionMode, newSubmissionMode),
changeSet)
queues = ceInfo['Queues'].keys()
for queue in queues:
queueInfo = ceInfo['Queues'][queue]
queueStatus = queueInfo['GlueCEStateStatus']
queueSection = cfgPath(ceSection, 'Queues', queue)
queueDict = {}
result = gConfig.getOptionsDict(queueSection)
if result['OK']:
queueDict = result['Value']
else:
if queueStatus.lower() == "production":
log.notice("Adding new queue",
"%s to CE %s" % (queue, ce))
else:
continue
# Current CS queue info
maxCPUTime = queueDict.get('maxCPUTime', 'Unknown')
si00 = queueDict.get('SI00', 'Unknown')
maxTotalJobs = queueDict.get('MaxTotalJobs', 'Unknown')
# Current BDII queue info
newMaxCPUTime = queueInfo.get('GlueCEPolicyMaxCPUTime', '')
if newMaxCPUTime == "4" * len(
newMaxCPUTime) or newMaxCPUTime == "9" * len(
newMaxCPUTime):
newMaxCPUTime = ''
wallTime = queueInfo.get('GlueCEPolicyMaxWallClockTime',
'')
if wallTime == "4" * len(
wallTime) or wallTime == "9" * len(wallTime):
wallTime = ''
if wallTime and int(wallTime) > 0:
if not newMaxCPUTime:
newMaxCPUTime = str(int(0.8 * int(wallTime)))
else:
if int(wallTime) <= int(newMaxCPUTime):
newMaxCPUTime = str(int(0.8 * int(wallTime)))
newSI00 = ''
caps = queueInfo.get('GlueCECapability', [])
if isinstance(caps, basestring):
caps = [caps]
for cap in caps:
if 'CPUScalingReferenceSI00' in cap:
newSI00 = cap.split('=')[-1]
# Adding queue info to the CS
addToChangeSet((queueSection, 'maxCPUTime', maxCPUTime,
newMaxCPUTime), changeSet)
addToChangeSet((queueSection, 'SI00', si00, newSI00),
changeSet)
if maxTotalJobs == "Unknown":
newTotalJobs = min(
1000,
int(
int(queueInfo.get('GlueCEInfoTotalCPUs', 0)) /
2))
newWaitingJobs = max(2, int(newTotalJobs * 0.1))
newTotalJobs = str(newTotalJobs)
newWaitingJobs = str(newWaitingJobs)
addToChangeSet(
(queueSection, 'MaxTotalJobs', '', newTotalJobs),
changeSet)
addToChangeSet((queueSection, 'MaxWaitingJobs', '',
newWaitingJobs), changeSet)
# Updating eligible VO list
VOs = set()
if queueDict.get('VO', ''):
VOs = set([
q.strip()
for q in queueDict.get('VO', '').split(',') if q
])
if vo not in VOs:
VOs.add(vo)
VOs = list(VOs)
newVOs = ','.join(VOs)
addToChangeSet((queueSection, 'VO', '', newVOs),
changeSet)
return S_OK(changeSet)
def checkUnusedSEs():
global vo, dry
result = getGridSRMs(vo, unUsed=True)
if not result["OK"]:
gLogger.error("Failed to look up SRMs in BDII", result["Message"])
siteSRMDict = result["Value"]
# Evaluate VOs
result = getVOs()
if result["OK"]:
csVOs = set(result["Value"])
else:
csVOs = {vo}
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]["SE"]
srmDict = siteSRMDict[site][gridSE]["SRM"]
# Check the SRM version
version = srmDict.get("GlueServiceVersion", "")
if not (version and version.startswith("2")):
gLogger.debug("Skipping SRM service with version %s" % version)
continue
result = getDIRACSiteName(site)
if not result["OK"]:
gLogger.notice("Unused se %s is detected at unused site %s" % (gridSE, site))
gLogger.notice("Consider adding site %s to the DIRAC CS" % site)
continue
diracSites = result["Value"]
yn = raw_input(
"\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: " % (gridSE, str(diracSites))
)
if not yn or yn.lower().startswith("y"):
if len(diracSites) > 1:
prompt = "Which DIRAC site the new SE should be attached to ?"
for i, s in enumerate(diracSites):
prompt += "\n[%d] %s" % (i, s)
prompt += "\nEnter your choice number: "
inp = raw_input(prompt)
try:
ind = int(inp)
except:
gLogger.notice("Can not interpret your choice: %s, try again later" % inp)
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split(".")
recName = "%s-disk" % siteName
inp = raw_input("Give a DIRAC name to the grid SE %s, default %s : " % (gridSE, recName))
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice("Adding new SE %s at site %s" % (diracSEName, diracSite))
seSection = cfgPath("/Resources/StorageElements", diracSEName)
changeSet.add((seSection, "BackendType", seDict.get("GlueSEImplementationName", "Unknown")))
changeSet.add((seSection, "Description", seDict.get("GlueSEName", "Unknown")))
bdiiVOs = set(
[re.sub("^VO:", "", rule) for rule in srmDict.get("GlueServiceAccessControlBaseRule", [])]
)
seVOs = csVOs.intersection(bdiiVOs)
changeSet.add((seSection, "VO", ",".join(seVOs)))
accessSection = cfgPath(seSection, "AccessProtocol.1")
changeSet.add((accessSection, "Protocol", "srm"))
changeSet.add((accessSection, "PluginName", "SRM2"))
endPoint = srmDict.get("GlueServiceEndpoint", "")
host = urlparse(endPoint).hostname
port = urlparse(endPoint).port
changeSet.add((accessSection, "Host", host))
changeSet.add((accessSection, "Port", port))
changeSet.add((accessSection, "Access", "remote"))
voPathSection = cfgPath(accessSection, "VOPath")
if "VOPath" in seDict:
path = seDict["VOPath"]
voFromPath = os.path.basename(path)
if voFromPath != diracVO:
gLogger.notice("\n!!! Warning: non-conventional VO path: %s\n" % path)
changeSet.add((voPathSection, diracVO, path))
path = os.path.dirname(path)
else:
# Try to guess the Path
domain = ".".join(host.split(".")[-2:])
path = "/dpm/%s/home" % domain
changeSet.add((accessSection, "Path", path))
changeSet.add((accessSection, "SpaceToken", ""))
changeSet.add((accessSection, "WSUrl", "/srm/managerv2?SFN="))
gLogger.notice("SE %s will be added with the following parameters" % diracSEName)
changeList = list(changeSet)
changeList.sort()
for entry in changeList:
gLogger.notice(entry)
yn = raw_input("Do you want to add new SE %s ? default yes [yes|no]: " % diracSEName)
if not yn or yn.lower().startswith("y"):
changeSetFull = changeSetFull.union(changeSet)
if dry:
if changeSetFull:
gLogger.notice("Skipping commit of the new SE data in a dry run")
else:
gLogger.notice("No new SE to be added")
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result["OK"]:
gLogger.error("Failed to initialize CSAPI object", result["Message"])
DIRACExit(-1)
changeList = list(changeSetFull)
changeList.sort()
for section, option, value in changeList:
csAPI.setOption(cfgPath(section, option), value)
yn = raw_input("New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ")
if not yn or yn.lower().startswith("y"):
result = csAPI.commit()
if not result["OK"]:
gLogger.error("Error while commit to CS", result["Message"])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSetFull))
else:
gLogger.notice("No new SE to be added")
return S_OK()
