def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/SAMManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'SAMManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def updateCS(changeSet):
global vo, dry, ceBdiiDict
changeList = sorted(changeSet)
if dry:
gLogger.notice("The following needed changes are detected:\n")
else:
gLogger.notice("We are about to make the following changes to CS:\n")
for entry in changeList:
gLogger.notice("%s/%s %s -> %s" % entry)
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result["OK"]:
gLogger.error("Failed to initialize CSAPI object", result["Message"])
DIRACExit(-1)
for section, option, value, new_value in changeSet:
if value == "Unknown" or not value:
csAPI.setOption(cfgPath(section, option), new_value)
else:
csAPI.modifyValue(cfgPath(section, option), new_value)
yn = six.moves.input("Do you want to commit changes to CS ? [default yes] [yes|no]: ")
if yn == "" or yn.lower().startswith("y"):
result = csAPI.commit()
if not result["OK"]:
gLogger.error("Error while commit to CS", result["Message"])
else:
gLogger.notice("Successfully committed %d changes to CS" % len(changeSet))
class TestComponentInstallation( unittest.TestCase ):
"""
Contains methods for testing of separate elements
"""
def setUp( self ):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI( self.host )
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result[ 'OK' ]:
raise Exception( result[ 'Message' ] )
cfg = result[ 'Value' ]
setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' )
self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' )
self.rootPwd = cfg.getOption( 'Systems/Databases/Password' )
self.diracPwd = self.rootPwd
def tearDown( self ):
pass
def initialize(self):
self.log = gLogger.getSubLogger(self.__class__.__name__)
# API initialization is required to get an up-to-date configuration from the CS
self.csAPI = CSAPI()
self.csAPI.initialize()
# temporary buffer for network accounting objects + some parameters
self.buffer = {
} # { {addTime: datetime.now(), object: Network() }, ... }
self.bufferTimeout = self.am_getOption("BufferTimeout",
NetworkAgent.BUFFER_TIMEOUT)
# internal list of message queue consumers
self.consumers = []
# host-to-dirac name dictionary
self.nameDictionary = {}
# statistics
self.messagesCount = 0 # number of received messages
self.messagesCountOld = 0 # previous number of received messages (used to check connection status)
self.skippedMessagesCount = 0 # number of skipped messages (errors, unsupported metrics, etc.)
self.PLRMetricCount = 0 # number of received packet-loss-rate metrics
self.OWDMetricCount = 0 # number of received one-way-delay metrics
self.skippedMetricCount = 0 # number of skipped metrics (errors, invalid data, etc.)
self.insertedCount = 0 # number of properly inserted accounting objects
self.removedCount = 0 # number of removed accounting objects (missing data)
return S_OK()
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(
"ShifterRole: Name of the shifter role, e.g. DataManager")
Script.registerArgument(
"UserName: A user name, as registered in Registry section")
Script.registerArgument(
"DIRACGroup: DIRAC Group, e.g. diracAdmin (the user has to have this role)"
)
Script.parseCommandLine(ignoreErrors=True)
csAPI = CSAPI()
shifterRole, userName, diracGroup = Script.getPositionalArgs(group=True)
res = csAPI.addShifter(
{shifterRole: {
"User": userName,
"Group": diracGroup
}})
if not res["OK"]:
gLogger.error("Could not add shifter", ": " + res["Message"])
DIRACExit(1)
res = csAPI.commit()
if not res["OK"]:
gLogger.error("Could not add shifter", ": " + res["Message"])
DIRACExit(1)
gLogger.notice("Added shifter %s as user %s with group %s" %
(shifterRole, userName, diracGroup))
def __setCSElementStatus(self, elementName, elementType, statusType, status):
"""
Sets on the CS the Elements status
"""
# DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing
if elementType in ('ComputingElement', 'FTS'):
return S_OK()
# If we are here it is because elementType is either 'StorageElement' or 'Catalog'
statuses = self.rssConfig.getConfigStatusType(elementType)
if statusType not in statuses:
gLogger.error("%s is not a valid statusType" % statusType)
return S_ERROR("%s is not a valid statusType: %s" % (statusType, statuses))
if elementType == 'StorageElement':
cs_path = "/Resources/StorageElements"
elif elementType == 'Catalog':
cs_path = "/Resources/FileCatalogs"
# FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs)
# but needs to be VO-aware
statusType = 'Status'
csAPI = CSAPI()
csAPI.setOption("%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType), status)
res = csAPI.commitChanges()
if not res['OK']:
gLogger.warn('CS: %s' % res['Message'])
return res
def initialize(self):
""" Initialize the default parameters
"""
voNames = self.am_getOption('VO', [])
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs(voNames)
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice("VOs: %s" % self.__voDict.keys())
self.csapi = CSAPI()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.autoAddUsers = self.am_getOption('AutoAddUsers',
self.autoAddUsers)
self.autoModifyUsers = self.am_getOption('AutoModifyUsers',
self.autoModifyUsers)
self.autoDeleteUsers = self.am_getOption('AutoDeleteUsers',
self.autoDeleteUsers)
self.detailedReport = self.am_getOption('DetailedReport',
self.detailedReport)
self.makeFCEntry = self.am_getOption('MakeHomeDirectory',
self.makeFCEntry)
return S_OK()
def updateCS( changeSet ):
global vo, dry, ceBdiiDict
changeList = list( changeSet )
changeList.sort()
if dry:
gLogger.notice( 'The following needed changes are detected:\n' )
else:
gLogger.notice( 'We are about to make the following changes to CS:\n' )
for entry in changeList:
gLogger.notice( "%s/%s %s -> %s" % entry )
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption( cfgPath( section, option ), new_value )
else:
csAPI.modifyValue( cfgPath( section, option ), new_value )
yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
if yn == '' or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
def main():
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
csAPI = CSAPI()
if len(args) < 3:
Script.showHelp(exitCode=1)
shifterRole = args[0]
userName = args[1]
diracGroup = args[2]
res = csAPI.addShifter(
{shifterRole: {
'User': userName,
'Group': diracGroup
}})
if not res['OK']:
gLogger.error("Could not add shifter", ": " + res['Message'])
DIRACExit(1)
res = csAPI.commit()
if not res['OK']:
gLogger.error("Could not add shifter", ": " + res['Message'])
DIRACExit(1)
gLogger.notice("Added shifter %s as user %s with group %s" %
(shifterRole, userName, diracGroup))
def __setCSElementStatus(self, elementName, elementType, statusType,
status):
"""
Sets on the CS the Elements status
"""
# DIRAC doesn't store the status of ComputingElements nor FTS in the CS, so here we can just do nothing
if elementType in ('ComputingElement', 'FTS'):
return S_OK()
# If we are here it is because elementType is either 'StorageElement' or 'Catalog'
statuses = self.rssConfig.getConfigStatusType(elementType)
if statusType not in statuses:
gLogger.error("%s is not a valid statusType" % statusType)
return S_ERROR("%s is not a valid statusType: %s" %
(statusType, statuses))
if elementType == 'StorageElement':
cs_path = "/Resources/StorageElements"
elif elementType == 'Catalog':
cs_path = "/Resources/FileCatalogs"
#FIXME: This a probably outdated location (new one is in /Operations/[]/Services/Catalogs)
# but needs to be VO-aware
statusType = 'Status'
csAPI = CSAPI()
csAPI.setOption(
"%s/%s/%s/%s" % (cs_path, elementName, elementType, statusType),
status)
res = csAPI.commitChanges()
if not res['OK']:
gLogger.warn('CS: %s' % res['Message'])
return res
def initialize(self):
"""Run at the agent initialization (normally every 500 cycles)"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption("DryRun", self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def updateCS( changeSet ):
global vo, dry, ceBdiiDict
changeList = list( changeSet )
changeList.sort()
if dry:
gLogger.notice( 'The following needed changes are detected:\n' )
else:
gLogger.notice( 'We are about to make the following changes to CS:\n' )
for entry in changeList:
gLogger.notice( "%s/%s %s -> %s" % entry )
if not dry:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
csAPI.setOption( cfgPath( section, option ), new_value )
else:
csAPI.modifyValue( cfgPath( section, option ), new_value )
yn = raw_input( 'Do you want to commit changes to CS ? [default yes] [yes|no]: ' )
if yn == '' or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSet ) )
def __init__(self):
"""initialise."""
CSAPI.__init__(self)
self._num_changes = 0
self._append_dict = ConfigDefaultDict(list)
self._append_unique_dict = ConfigDefaultDict(set)
result = self.initialize()
if not result['OK']:
gLogger.error('Failed to initialise CSAPI object:',
result['Message'])
raise RuntimeError(result['Message'])
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False):
self.log = gLogger.getSubLogger("VOMS2CSSynchronizer")
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {'Errors': [], 'Info': []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs',
self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
def updateFlag():
# Here now setting the flag as the following inside /Operations/Defaults:
# in Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
res = csAPI.createSection('Operations/Defaults/Services/')
if not res['OK']:
print(res['Message'])
exit(1)
res = csAPI.createSection('Operations/Defaults/Services/JobMonitoring/')
if not res['OK']:
print(res['Message'])
exit(1)
csAPI.setOption(
'Operations/Defaults/Services/JobMonitoring/useESForJobParametersFlag',
True)
csAPI.commit()
# Now we need to restart the services for the new configuration to be picked up
time.sleep(2)
os.system("dirac-restart-component WorkloadManagement JobMonitoring")
os.system("dirac-restart-component WorkloadManagement JobStateUpdate")
time.sleep(5)
def __init__(self):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.detmodels = {}
self.lcgeo_env = "lcgeo_DIR"
self.ddhep_env = "DD4HEP"
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.version = ''
self.platform = 'x86_64-slc5-gcc43-opt'
self.comment = ""
self.name = "ddsim"
self.csapi = CSAPI()
self.tarBallName = None
self.md5sum = None
def getMQParamsFromCS(mqURI):
"""Function gets parameters of a MQ destination (queue/topic) from the CS.
Args:
mqURI(str):Pseudo URI identifing the MQ service. It has the following format:
mqConnection::DestinationType::DestinationName
e.g. blabla.cern.ch::Queues::MyQueue1
mType(str): 'consumer' or 'producer'
Returns:
S_OK(param_dicts) or S_ERROR
"""
# API initialization is required to get an up-to-date configuration from the CS
csAPI = CSAPI()
csAPI.initialize()
try:
mqService, mqType, mqName = mqURI.split("::")
except ValueError:
return S_ERROR("Bad format of mqURI address:%s" % (mqURI))
result = gConfig.getConfigurationTree("/Resources/MQServices", mqService,
mqType, mqName)
if not result["OK"] or not result["Value"]:
return S_ERROR(
"Requested destination not found in the CS: %s::%s::%s" %
(mqService, mqType, mqName))
mqDestinationPath = None
for path, value in result["Value"].items():
if not value and path.endswith(mqName):
mqDestinationPath = path
# set-up internal parameter depending on the destination type
tmp = mqDestinationPath.split("Queues")[0].split("Topics")
servicePath = tmp[0]
serviceDict = {}
if len(tmp) > 1:
serviceDict["Topic"] = mqName
else:
serviceDict["Queue"] = mqName
result = gConfig.getOptionsDict(servicePath)
if not result["OK"]:
return result
serviceDict.update(result["Value"])
result = gConfig.getOptionsDict(mqDestinationPath)
if not result["OK"]:
return result
serviceDict.update(result["Value"])
return S_OK(serviceDict)
def __init__(self, cliParams):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.modifiedCS = False
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.mailadress = '*****@*****.**'
self.cliParams = cliParams
self.parameter = dict(softSec=self.softSec,
platform=cliParams.platform,
version=cliParams.version,
basepath=cliParams.basePath,
initsctipt=cliParams.initScriptLocation)
self.applications = cliParams.applicationSet
self.detmodels = {}
self.csAPI = CSAPI()
def initialize(self):
"""Gets run paramaters from the configuration"""
self.addressTo = self.am_getOption("MailTo", self.addressTo)
self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption("AlternativeBDIIs",
self.alternativeBDIIs)
self.host = self.am_getOption("Host", self.host)
self.injectSingleCoreQueues = self.am_getOption(
"InjectSingleCoreQueues", self.injectSingleCoreQueues)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(":")[-1].isdigit():
self.alternativeBDIIs[index] += ":2170"
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption("ProcessCEs", self.processCEs)
self.selectedSites = self.am_getOption("SelectedSites", [])
self.dryRun = self.am_getOption("DryRun", self.dryRun)
self.voName = self.am_getOption("VirtualOrganization", self.voName)
if not self.voName:
self.voName = self.am_getOption("VO", [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == "all"):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result["OK"]:
vos = result["Value"]
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def __init__(self):
"""Internal initialization of the DIRAC Admin API."""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG":
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + "/ScratchDir",
"/tmp")
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'TestManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def beginExecution(self):
"""Reload the configurations before every cycle."""
self.setup = self.am_getOption("Setup", self.setup)
self.enabled = self.am_getOption("EnableFlag", self.enabled)
self.restartAgents = self.am_getOption("RestartAgents", self.restartAgents)
self.restartExecutors = self.am_getOption("RestartExecutors", self.restartExecutors)
self.restartServices = self.am_getOption("RestartServices", self.restartServices)
self.diracLocation = os.environ.get("DIRAC", self.diracLocation)
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
self.controlComponents = self.am_getOption('ControlComponents', self.controlComponents)
self.commitURLs = self.am_getOption('CommitURLs', self.commitURLs)
self.csAPI = CSAPI()
res = self.getRunningInstances(instanceType='Agents')
if not res["OK"]:
return S_ERROR("Failure to get running agents")
self.agents = res["Value"]
res = self.getRunningInstances(instanceType='Executors')
if not res["OK"]:
return S_ERROR("Failure to get running executors")
self.executors = res["Value"]
res = self.getRunningInstances(instanceType='Services')
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
self.accounting.clear()
return S_OK()
def initialize( self ):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.StorageElementToBeChecked = Queue.Queue()
self.StorageElementInCheck = []
self.maxNumberOfThreads = self.am_getOption( 'maxThreadsInPool', 1 )
self.threadPool = ThreadPool( self.maxNumberOfThreads,
self.maxNumberOfThreads )
if not self.threadPool:
self.log.error( 'Can not create Thread Pool' )
return S_ERROR( 'Can not create Thread Pool' )
self.setup = getSetup()[ 'Value' ]
self.VOExtension = getExt()
self.StorageElsWriteFreqs = CheckingFreqs[ 'StorageElsWriteFreqs' ]
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for _i in xrange( self.maxNumberOfThreads ):
self.threadPool.generateJobAndQueueIt( self._executeCheck, args = ( None, ) )
return S_OK()
except Exception:
errorStr = "StElWriteInspectorAgent initialization"
gLogger.exception( errorStr )
return S_ERROR( errorStr )
def insertCSSection( self, path, pardict ):
""" insert a section and values (or subsections) into the CS
:param str path: full path of the new section
:param str pardict: dictionary of key values in the new section, values can also be dictionaries
:return: S_OK(), S_ERROR()
"""
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
if self.csapi is None:
self.csapi = CSAPI()
for key, value in pardict.iteritems():
newSectionPath = os.path.join(path,key)
gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) )
self.csapi.createSection( path )
if isinstance( value, dict ):
res = self.insertCSSection( newSectionPath, value )
else:
res = self.csapi.setOption( newSectionPath, value )
if not res['OK']:
return res
else:
gLogger.notice( "Added to CS: %s " % res['Value'] )
return S_OK("Added all things to cs")
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + '/LogLevel', 'DEBUG') == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + '/ScratchDir',
'/tmp')
self.currentDir = os.getcwd()
self.rssFlag = ResourceStatus().rssFlag
self.sitestatus = SiteStatus()
self._siteSet = set(getSites().get('Value', []))
def initialize( self ):
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def __init__( self, agentName, baseAgentName, properties ): AgentModule.__init__( self, agentName, baseAgentName, properties ) self.addressTo = '' self.addressFrom = '' self.voName = '' self.csAPI = CSAPI() self.subject = "CE2CSAgent" self.alternativeBDIIs = []
def __init__(
self,
vo,
autoModifyUsers=True,
autoAddUsers=True,
autoDeleteUsers=False,
autoLiftSuspendedStatus=False,
syncPluginName=None,
):
"""VOMS2CSSynchronizer class constructor
:param str vo: VO to be synced
:param boolean autoModifyUsers: flag to automatically modify user data in CS
:param autoAddUsers: flag to automatically add new users to CS
:param autoDeleteUsers: flag to automatically delete users from CS if no more in VOMS
:param autoLiftSuspendedStatus: flag to automatically remove Suspended status in CS
:param syncPluginName: name of the plugin to validate or extend users' info
:return: None
"""
self.log = gLogger.getSubLogger(self.__class__.__name__)
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {"Errors": [], "Info": []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
self.autoLiftSuspendedStatus = autoLiftSuspendedStatus
self.voChanged = False
self.syncPlugin = None
if syncPluginName:
objLoader = ObjectLoader()
_class = objLoader.loadObject(
"ConfigurationSystem.Client.SyncPlugins.%sSyncPlugin" % syncPluginName, "%sSyncPlugin" % syncPluginName
)
if not _class["OK"]:
raise Exception(_class["Message"])
self.syncPlugin = _class["Value"]()
def getMQParamsFromCS( mqURI ):
""" Function gets parameters of a MQ destination (queue/topic) from the CS.
Args:
mqURI(str):Pseudo URI identifing the MQ service. It has the following format:
mqConnection::DestinationType::DestinationName
e.g. blabla.cern.ch::Queue::MyQueue1
mType(str): 'consumer' or 'producer'
Returns:
S_OK(param_dicts)/S_ERROR:
"""
# API initialization is required to get an up-to-date configuration from the CS
csAPI = CSAPI()
csAPI.initialize()
try :
mqService, mqType, mqName = mqURI.split( "::" )
except ValueError:
return S_ERROR( 'Bad format of mqURI address:%s' % ( mqURI ) )
result = gConfig.getConfigurationTree( '/Resources/MQServices', mqService, mqType, mqName )
if not result['OK'] or len( result['Value'] ) == 0:
return S_ERROR( 'Requested destination not found in the CS: %s::%s::%s' % ( mqService, mqType, mqName ) )
mqDestinationPath = None
for path, value in result['Value'].iteritems():
if not value and path.endswith( mqName ):
mqDestinationPath = path
# set-up internal parameter depending on the destination type
tmp = mqDestinationPath.split( 'Queue' )[0].split( 'Topic' )
servicePath = tmp[0]
serviceDict = {}
if len( tmp ) > 1:
serviceDict['Topic'] = mqName
else:
serviceDict['Queue'] = mqName
result = gConfig.getOptionsDict( servicePath )
if not result['OK']:
return result
serviceDict.update( result['Value'] )
result = gConfig.getOptionsDict( mqDestinationPath )
if not result['OK']:
return result
serviceDict.update( result['Value'] )
return S_OK( serviceDict )
def initialize(self):
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', [])
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.subject = "Bdii2CSAgent"
self.processCEs = self.am_getOption('ProcessCEs', True)
self.processSEs = self.am_getOption('ProcessSEs', False)
self.voName = self.am_getOption('VirtualOrganization', [])
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1
and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result['OK']:
raise Exception(result['Message'])
chain = result['Value']['chain']
result = chain.getCertInChain(-1)
if not result['OK']:
raise Exception(result['Message'])
result = result['Value'].getSubjectDN()
if not result['OK']:
raise Exception(result['Message'])
userDN = result['Value']
result = getUsernameForDN(userDN)
if not result['OK']:
raise Exception(result['Message'])
self.user = result['Value']
if not self.user:
self.user = '******'
def tearDown(self):
pass
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = "localhost"
self.notificationPort = 9154
self.rootPwd = ""
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result["OK"]:
raise Exception(result["Message"])
cfg = result["Value"]
setup = cfg.getOption("DIRAC/Setup", "dirac-JenkinsSetup")
self.frameworkSetup = cfg.getOption("DIRAC/Setups/" + setup +
"/Framework")
self.rootPwd = cfg.getOption("Systems/Databases/Password")
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result["OK"]:
raise Exception(result["Message"])
chain = result["Value"]["chain"]
result = chain.getCertInChain(-1)
if not result["OK"]:
raise Exception(result["Message"])
result = result["Value"].getSubjectDN()
if not result["OK"]:
raise Exception(result["Message"])
userDN = result["Value"]
result = getUsernameForDN(userDN)
if not result["OK"]:
raise Exception(result["Message"])
self.user = result["Value"]
if not self.user:
self.user = "******"
def tearDown(self):
pass
class TestComponentInstallation(unittest.TestCase):
"""
Contains methods for testing of separate elements
"""
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'dirac-JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup + '/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
result = getProxyInfo()
if not result['OK']:
raise Exception(result['Message'])
chain = result['Value']['chain']
result = chain.getCertInChain(-1)
if not result['OK']:
raise Exception(result['Message'])
result = result['Value'].getSubjectDN()
if not result['OK']:
raise Exception(result['Message'])
userDN = result['Value']
result = getUsernameForDN(userDN)
if not result['OK']:
raise Exception(result['Message'])
self.user = result['Value']
if not self.user:
self.user = '******'
def tearDown(self):
pass
def initialize(self):
""" Run at the agent initialization (normally every 500 cycles)
"""
# client to connect to GOCDB
self.GOCDBClient = GOCDBClient()
self.dryRun = self.am_getOption('DryRun', self.dryRun)
# API needed to update configuration stored by CS
self.csAPI = CSAPI()
return self.csAPI.initialize()
def setUp(self):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI(self.host)
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result['OK']:
raise Exception(result['Message'])
cfg = result['Value']
setup = cfg.getOption('DIRAC/Setup', 'JenkinsSetup')
self.frameworkSetup = cfg.getOption('DIRAC/Setups/' + setup +
'/Framework')
self.rootPwd = cfg.getOption('Systems/Databases/Password')
self.diracPwd = self.rootPwd
def __setCSStorageElementStatus( self, elementName, statusType, status ):
"""
Sets on the CS the StorageElements status
"""
statuses = self.rssConfig.getConfigStatusType( 'StorageElement' )
if not statusType in statuses:
gLogger.error( "%s is not a valid statusType" % statusType )
return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) )
csAPI = CSAPI()
cs_path = "/Resources/StorageElements"
csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status )
res = csAPI.commitChanges()
if not res[ 'OK' ]:
gLogger.warn( 'CS: %s' % res[ 'Message' ] )
return res
def __setCSStorageElementStatus( self, elementName, statusType, status ):
"""
Sets on the CS the StorageElements status
"""
statuses = self.rssConfig.getConfigStatusType( 'StorageElement' )
if not statusType in statuses:
gLogger.error( "%s is not a valid statusType" % statusType )
return S_ERROR( "%s is not a valid statusType: %s" % ( statusType, statuses ) )
csAPI = CSAPI()
cs_path = "/Resources/StorageElements"
csAPI.setOption( "%s/%s/%s" % ( cs_path, elementName, statusType ), status )
res = csAPI.commitChanges()
if not res[ 'OK' ]:
gLogger.warn( 'CS: %s' % res[ 'Message' ] )
return res
def test_configurationAutoUpdate(value1, value2):
"""
Test if service refresh his configuration. It sent a random value to the CS
and check if Service can return it.
"""
csapi = CSAPI()
# SETTING FIRST VALUE
csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value1)
csapi.commitChanges()
# Wait for automatic refresh (+1 to be sure that request is done)
time.sleep(gConfigurationData.getPropagationTime() + 1)
RPCClient("Framework/User").getTestValue()
assert RPCClient("Framework/User").getTestValue()["Value"] == value1
# SETTING SECOND VALUE
csapi.modifyValue("/DIRAC/Configuration/TestUpdateValue", value2)
csapi.commitChanges()
time.sleep(gConfigurationData.getPropagationTime() + 1)
assert RPCClient("Framework/User").getTestValue()["Value"] == value2
def __init__( self ):
"""Internal initialization of the DIRAC Admin API.
"""
super( DiracAdmin, self ).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue( self.section + '/LogLevel', 'DEBUG' ) == 'DEBUG':
self.dbg = True
self.scratchDir = gConfig.getValue( self.section + '/ScratchDir', '/tmp' )
self.currentDir = os.getcwd()
def __init__(self):
"""Internal initialization of the DIRAC Admin API.
"""
super(DiracAdmin, self).__init__()
self.csAPI = CSAPI()
self.dbg = False
if gConfig.getValue(self.section + "/LogLevel", "DEBUG") == "DEBUG":
self.dbg = True
self.scratchDir = gConfig.getValue(self.section + "/ScratchDir", "/tmp")
self.currentDir = os.getcwd()
def __init__(self, vo, autoModifyUsers=True, autoAddUsers=True, autoDeleteUsers=False):
self.log = gLogger.getSubLogger("VOMS2CSSynchronizer")
self.csapi = CSAPI()
self.vo = vo
self.vomsVOName = getVOOption(vo, "VOMSName", "")
if not self.vomsVOName:
raise Exception("VOMS name not defined for VO %s" % vo)
self.adminMsgs = {'Errors': [], 'Info': []}
self.vomsUserDict = {}
self.autoModifyUsers = autoModifyUsers
self.autoAddUsers = autoAddUsers
self.autoDeleteUsers = autoDeleteUsers
def __init__( self ):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.detmodels = {}
self.lcgeo_env="lcgeo_DIR"
self.ddhep_env="DD4HEP"
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.version = ''
self.platform = 'x86_64-slc5-gcc43-opt'
self.comment = ""
self.name = "ddsim"
self.csapi = CSAPI()
self.tarBallName = None
self.md5sum = None
def beginExecution(self):
"""Reload the configurations before every cycle."""
self.setup = self.am_getOption("Setup", self.setup)
self.enabled = self.am_getOption("EnableFlag", self.enabled)
self.restartAgents = self.am_getOption("RestartAgents",
self.restartAgents)
self.restartExecutors = self.am_getOption("RestartExecutors",
self.restartExecutors)
self.restartServices = self.am_getOption("RestartServices",
self.restartServices)
self.addressTo = self.am_getOption("MailTo", self.addressTo)
self.addressFrom = self.am_getOption("MailFrom", self.addressFrom)
self.controlComponents = self.am_getOption("ControlComponents",
self.controlComponents)
self.commitURLs = self.am_getOption("CommitURLs", self.commitURLs)
self.doNotRestartInstancePattern = self.am_getOption(
"DoNotRestartInstancePattern", self.doNotRestartInstancePattern)
self.csAPI = CSAPI()
res = self.getRunningInstances(instanceType="Agents")
if not res["OK"]:
return S_ERROR("Failure to get running agents")
self.agents = res["Value"]
res = self.getRunningInstances(instanceType="Executors")
if not res["OK"]:
return S_ERROR("Failure to get running executors")
self.executors = res["Value"]
res = self.getRunningInstances(instanceType="Services")
if not res["OK"]:
return S_ERROR("Failure to get running services")
self.services = res["Value"]
self.accounting.clear()
return S_OK()
def initialize(self):
""" Gets run paramaters from the configuration
"""
self.addressTo = self.am_getOption('MailTo', self.addressTo)
self.addressFrom = self.am_getOption('MailFrom', self.addressFrom)
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption('AlternativeBDIIs', self.alternativeBDIIs)
self.host = self.am_getOption('Host', self.host)
self.glue2URLs = self.am_getOption('GLUE2URLs', self.glue2URLs)
self.glue2Only = self.am_getOption('GLUE2Only', self.glue2Only)
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate(self.alternativeBDIIs):
if not url.split(':')[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info("MailTo", self.addressTo)
self.log.info("MailFrom", self.addressFrom)
if self.alternativeBDIIs:
self.log.info("AlternativeBDII URLs:", self.alternativeBDIIs)
self.processCEs = self.am_getOption('ProcessCEs', self.processCEs)
self.processSEs = self.am_getOption('ProcessSEs', self.processSEs)
self.selectedSites = self.am_getOption('SelectedSites', [])
self.dryRun = self.am_getOption('DryRun', self.dryRun)
self.voName = self.am_getOption('VirtualOrganization', self.voName)
if not self.voName:
self.voName = self.am_getOption('VO', [])
if not self.voName or (len(self.voName) == 1 and self.voName[0].lower() == 'all'):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption(vo, "VOMSName")
if vomsVO:
self.voName.append(vomsVO)
if self.voName:
self.log.info("Agent will manage VO(s) %s" % self.voName)
else:
self.log.fatal("VirtualOrganization option not defined for agent")
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def __init__(self, cliParams ):
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
self.modifiedCS = False
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.mailadress = '*****@*****.**'
self.cliParams = cliParams
self.parameter = dict( softSec = self.softSec,
platform = cliParams.platform,
version = cliParams.version,
basepath = cliParams.basePath,
initsctipt = cliParams.initScriptLocation
)
self.applications = cliParams.applicationSet
self.detmodels = {}
self.csAPI = CSAPI()
def initialize(self):
""" Standard constructor
"""
try:
self.rsDB = ResourceStatusDB()
self.rmDB = ResourceManagementDB()
self.ResourcesToBeChecked = Queue.Queue()
self.ResourceNamesInCheck = []
self.maxNumberOfThreads = self.am_getOption('maxThreadsInPool', 1)
self.threadPool = ThreadPool(self.maxNumberOfThreads,
self.maxNumberOfThreads)
if not self.threadPool:
self.log.error('Can not create Thread Pool')
return S_ERROR('Can not create Thread Pool')
self.setup = getSetup()['Value']
self.VOExtension = getExt()
configModule = __import__(
self.VOExtension +
"DIRAC.ResourceStatusSystem.Policy.Configurations", globals(),
locals(), ['*'])
self.Resources_check_freq = copy.deepcopy(
configModule.Resources_check_freq)
self.nc = NotificationClient()
self.diracAdmin = DiracAdmin()
self.csAPI = CSAPI()
for i in xrange(self.maxNumberOfThreads):
self.threadPool.generateJobAndQueueIt(self._executeCheck,
args=(None, ))
return S_OK()
except Exception:
errorStr = "RSInspectorAgent initialization"
gLogger.exception(errorStr)
return S_ERROR(errorStr)
def setUp(self):
self.dirac = Dirac()
csAPI = CSAPI()
self.lfn5 = os.path.join(DESTINATION_PATH, "test_file_10MB_v5.bin")
self.dir5 = os.path.dirname(self.lfn5)
# local file, for now:
self.fname = os.path.basename(self.lfn5)
random_dd(self.fname, 10)
self.diracSE = "SE-1"
try:
self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog")
except Exception:
self.fail(" FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n" + traceback.format_exc())
return
# add a replica
self.fileadded = self.dirac.addFile(self.lfn5, self.fname, self.diracSE)
self.assertTrue(self.fileadded["OK"])
def initialize( self ):
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
self.processCEs = self.am_getOption( 'ProcessCEs', True )
self.processSEs = self.am_getOption( 'ProcessSEs', False )
self.voName = self.am_getOption( 'VirtualOrganization', [] )
if not self.voName:
self.voName = self.am_getOption( 'VO', [] )
if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption( vo, "VOMSName" )
if vomsVO:
self.voName.append( vomsVO )
if self.voName:
self.log.info( "Agent will manage VO(s) %s" % self.voName )
else:
self.log.fatal( "VirtualOrganization option not defined for agent" )
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
def setUp( self ):
self.host = 'localhost'
self.notificationPort = 9154
self.rootPwd = ''
self.csClient = CSAPI()
self.monitoringClient = ComponentMonitoringClient()
self.client = SystemAdministratorClientCLI( self.host )
self.csClient.downloadCSData()
result = self.csClient.getCurrentCFG()
if not result[ 'OK' ]:
raise Exception( result[ 'Message' ] )
cfg = result[ 'Value' ]
setup = cfg.getOption( 'DIRAC/Setup', 'JenkinsSetup' )
self.frameworkSetup = cfg.getOption( 'DIRAC/Setups/' + setup + '/Framework' )
self.rootPwd = cfg.getOption( 'Systems/Databases/Password' )
self.diracPwd = self.rootPwd
def initialize( self ):
""" Initialize the default parameters
"""
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.csapi = CSAPI()
self.dryRun = self.am_getOption( 'DryRun', self.dryRun )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', self.autoAddUsers )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', self.autoModifyUsers )
return S_OK()
def commit(self):
"""Commit the changes to the configuration system."""
# Perform all the appending operations at the end to only get from current config once.
for path, value in chain(self._append_dict.iteritems(),
self._append_unique_dict.iteritems()):
section, option = path.rsplit('/', 1)
self.add(section, option, value)
self._num_changes += 1
result = CSAPI.commit(self)
if not result['OK']:
gLogger.error("Error while commit to CS", result['Message'])
raise RuntimeError("Error while commit to CS")
if self._num_changes:
gLogger.notice("Successfully committed %d changes to CS\n"
% self._num_changes)
self._num_changes = 0
self._append_dict.clear()
self._append_unique_dict.clear()
else:
gLogger.notice("No changes to commit")
def initialize( self ):
self.__voDict = {}
voNames = self.am_getOption( 'VO', [] )
if not voNames[0].lower() == "none":
if voNames[0].lower() == "any":
voNames = []
result = getVOMSVOs( voNames )
if not result['OK']:
return result
self.__voDict = result['Value']
self.__adminMsgs = {}
self.csapi = CSAPI()
self.voChanged = False
self.log.notice( "VOs: %s" % self.__voDict.keys() )
self.autoAddUsers = self.am_getOption( 'AutoAddUsers', False )
self.autoModifyUsers = self.am_getOption( 'AutoModifyUsers', False )
self.autoSuspendUsers = self.am_getOption( 'AutoSuspendUsers', False )
return S_OK()
'Usage:',
' %s [option|cfgFile] ... DB ...' % Script.scriptName] ) )
Script.parseCommandLine()
args = Script.getPositionalArgs()
setupName = args[0]
# Where to store outputs
if not os.path.isdir( '%s/sandboxes' % setupName ):
os.makedirs( '%s/sandboxes' % setupName )
# now updating the CS
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
csAPI = CSAPI()
csAPI.setOption( 'Systems/WorkloadManagement/Production/Services/SandboxStore/BasePath', '%s/sandboxes' % setupName )
csAPI.setOption( 'Systems/WorkloadManagement/Production/Services/SandboxStore/LogLevel', 'DEBUG' )
# Now setting a SandboxSE as the following:
# ProductionSandboxSE
# {
# BackendType = DISET
# AccessProtocol.1
# {
# Host = localhost
# Port = 9196
# ProtocolName = DIP
# Protocol = dips
# Path = /scratch/workspace/%s/sandboxes % setupName
"\n".join(
[
__doc__.split("\n")[1],
"Usage:",
" %s [option|cfgfile] ... DIRACSiteName GridSiteName CE [CE] ..." % Script.scriptName,
"Arguments:",
" DIRACSiteName: Name of the site for DIRAC in the form GRID.LOCATION.COUNTRY (ie:LCG.CERN.ch)",
" GridSiteName: Name of the site in the Grid (ie: CERN-PROD)",
" CE: Name of the CE to be included in the site (ie: ce111.cern.ch)",
]
)
)
Script.parseCommandLine(ignoreErrors=True)
args = Script.getPositionalArgs()
csAPI = CSAPI()
if len(args) < 3:
Script.showHelp()
DIRACExit(-1)
diracSiteName = args[0]
gridSiteName = args[1]
ces = args[2:]
try:
diracGridType, place, country = diracSiteName.split(".")
except ValueError:
gLogger.error("The DIRACSiteName should be of the form GRID.LOCATION.COUNTRY for example LCG.CERN.ch")
DIRACExit(-1)
result = getDIRACSiteName(gridSiteName)
class Bdii2CSAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize( self ):
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# Create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# Check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
self.processCEs = self.am_getOption( 'ProcessCEs', True )
self.processSEs = self.am_getOption( 'ProcessSEs', False )
self.voName = self.am_getOption( 'VirtualOrganization', [] )
if not self.voName:
self.voName = self.am_getOption( 'VO', [] )
if not self.voName or ( len( self.voName ) == 1 and self.voName[0].lower() == 'all' ):
# Get all VOs defined in the configuration
self.voName = []
result = getVOs()
if result['OK']:
vos = result['Value']
for vo in vos:
vomsVO = getVOOption( vo, "VOMSName" )
if vomsVO:
self.voName.append( vomsVO )
if self.voName:
self.log.info( "Agent will manage VO(s) %s" % self.voName )
else:
self.log.fatal( "VirtualOrganization option not defined for agent" )
return S_ERROR()
self.voBdiiCEDict = {}
self.voBdiiSEDict = {}
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
""" General agent execution method
"""
# Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result['OK']:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
if self.processCEs:
self.__lookForNewCEs()
self.__updateCEs()
if self.processSEs:
self.__lookForNewSEs()
self.__updateSEs()
return S_OK()
def __lookForNewCEs( self ):
""" Look up BDII for CEs not yet present in the DIRAC CS
"""
bannedCEs = self.am_getOption( 'BannedCEs', [] )
result = getCEsFromCS()
if not result['OK']:
return result
knownCEs = set( result['Value'] )
knownCEs = knownCEs.union( set( bannedCEs ) )
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridCEs( vo, bdiiInfo = bdiiInfo, ceBlackList = knownCEs )
if not result['OK']:
self.log.error( 'Failed to get unused CEs', result['Message'] )
siteDict = result['Value']
body = ''
for site in siteDict:
newCEs = set( siteDict[site].keys() )
if not newCEs:
continue
ceString = ''
for ce in newCEs:
queueString = ''
ceInfo = bdiiInfo[site]['CEs'][ce]
ceString = "CE: %s, GOCDB Site Name: %s" % ( ce, site )
systemTuple = siteDict[site][ce]['System']
osString = "%s_%s_%s" % ( systemTuple )
newCEString = "\n%s\n%s\n" % ( ceString, osString )
for queue in ceInfo['Queues']:
queueStatus = ceInfo['Queues'][queue].get( 'GlueCEStateStatus', 'UnknownStatus' )
if 'production' in queueStatus.lower():
ceType = ceInfo['Queues'][queue].get( 'GlueCEImplementationName', '' )
queueString += " %s %s %s\n" % ( queue, queueStatus, ceType )
if queueString:
ceString = newCEString
ceString += "Queues:\n"
ceString += queueString
if ceString:
body += ceString
if body:
body = "\nWe are glad to inform You about new CE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list.\n"
body += "Add new Sites/CEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --ce\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def __getBdiiCEInfo( self, vo ):
if vo in self.voBdiiCEDict:
return S_OK( self.voBdiiCEDict[vo] )
self.log.info( "Check for available CEs for VO", vo )
result = getBdiiCEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiCEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiCEDict[vo] = result['Value']
return result
def __getBdiiSEInfo( self, vo ):
if vo in self.voBdiiSEDict:
return S_OK( self.voBdiiSEDict[vo] )
self.log.info( "Check for available SEs for VO", vo )
result = getBdiiSEInfo( vo )
message = ''
if not result['OK']:
message = result['Message']
for bdii in self.alternativeBDIIs :
result = getBdiiSEInfo( vo, host = bdii )
if result['OK']:
break
if not result['OK']:
if message:
self.log.error( "Error during BDII request", message )
else:
self.log.error( "Error during BDII request", result['Message'] )
else:
self.voBdiiSEDict[vo] = result['Value']
return result
def __updateCEs( self ):
""" Update the Site/CE/queue settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiCEInfo( vo )
if not result['OK']:
continue
ceBdiiDict = result['Value']
result = getSiteUpdates( vo, bdiiInfo = ceBdiiDict, log = self.log )
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union( result['Value'] )
# We have collected all the changes, consolidate VO settings
result = self.__updateCS( bdiiChangeSet )
return result
def __updateCS( self, bdiiChangeSet ):
queueVODict = {}
changeSet = set()
for entry in bdiiChangeSet:
section, option , _value, new_value = entry
if option == "VO":
queueVODict.setdefault( section, set() )
queueVODict[section] = queueVODict[section].union( set( new_value.split( ',' ) ) )
else:
changeSet.add( entry )
for section, VOs in queueVODict.items():
changeSet.add( ( section, 'VO', '', ','.join( VOs ) ) )
if changeSet:
changeList = list( changeSet )
changeList.sort()
body = '\n'.join( [ "%s/%s %s -> %s" % entry for entry in changeList ] )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if body:
self.log.info( 'The following configuration changes were detected:' )
self.log.info( body )
for section, option, value, new_value in changeSet:
if value == 'Unknown' or not value:
self.csAPI.setOption( cfgPath( section, option ), new_value )
else:
self.csAPI.modifyValue( cfgPath( section, option ), new_value )
result = self.csAPI.commit()
if not result['OK']:
self.log.error( "Error while committing to CS", result['Message'] )
else:
self.log.info( "Successfully committed %d changes to CS" % len( changeList ) )
return result
else:
self.log.info( "No changes found" )
return S_OK()
def __lookForNewSEs( self ):
""" Look up BDII for SEs not yet present in the DIRAC CS
"""
bannedSEs = self.am_getOption( 'BannedSEs', [] )
result = getSEsFromCS()
if not result['OK']:
return result
knownSEs = set( result['Value'] )
knownSEs = knownSEs.union( set( bannedSEs ) )
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
bdiiInfo = result['Value']
result = getGridSRMs( vo, bdiiInfo = bdiiInfo, srmBlackList = knownSEs )
if not result['OK']:
continue
siteDict = result['Value']
body = ''
for site in siteDict:
newSEs = set( siteDict[site].keys() )
if not newSEs:
continue
for se in newSEs:
body += '\n New SE %s available at site %s:\n' % ( se, site )
backend = siteDict[site][se]['SE'].get( 'GlueSEImplementationName', 'Unknown' )
size = siteDict[site][se]['SE'].get( 'GlueSESizeTotal', 'Unknown' )
body += ' Backend %s, Size %s' % ( backend, size )
if body:
body = "\nWe are glad to inform You about new SE(s) possibly suitable for %s:\n" % vo + body
body += "\n\nTo suppress information about an SE add its name to BannedSEs list.\n"
body += "Add new SEs for vo %s with the command:\n" % vo
body += "dirac-admin-add-resources --vo %s --se\n" % vo
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
if not result['OK']:
self.log.error( 'Can not send new site notification mail', result['Message'] )
return S_OK()
def __updateSEs( self ):
""" Update the Storage Element settings in the CS if they were changed in the BDII
"""
bdiiChangeSet = set()
for vo in self.voName:
result = self.__getBdiiSEInfo( vo )
if not result['OK']:
continue
seBdiiDict = result['Value']
result = getSRMUpdates( vo, bdiiInfo = seBdiiDict )
if not result['OK']:
continue
bdiiChangeSet = bdiiChangeSet.union( result['Value'] )
# We have collected all the changes, consolidate VO settings
result = self.__updateCS( bdiiChangeSet )
return result
class CE2CSAgent( AgentModule ):
addressTo = ''
addressFrom = ''
voName = ''
subject = "CE2CSAgent"
alternativeBDIIs = []
def initialize( self ):
# TODO: Have no default and if no mail is found then use the diracAdmin group
# and resolve all associated mail addresses.
self.addressTo = self.am_getOption( 'MailTo', self.addressTo )
self.addressFrom = self.am_getOption( 'MailFrom', self.addressFrom )
# create a list of alternative bdii urls
self.alternativeBDIIs = self.am_getOption( 'AlternativeBDIIs', [] )
# check if the bdii url is appended by a port number, if not append the default 2170
for index, url in enumerate( self.alternativeBDIIs ):
if not url.split( ':' )[-1].isdigit():
self.alternativeBDIIs[index] += ':2170'
if self.addressTo and self.addressFrom:
self.log.info( "MailTo", self.addressTo )
self.log.info( "MailFrom", self.addressFrom )
if self.alternativeBDIIs :
self.log.info( "AlternativeBDII URLs:", self.alternativeBDIIs )
self.subject = "CE2CSAgent"
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/TestManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'TestManager' )
self.voName = self.am_getOption( 'VirtualOrganization', self.voName )
if not self.voName:
self.voName = getVO()
if not self.voName:
self.log.fatal( "VO option not defined for agent" )
return S_ERROR()
self.csAPI = CSAPI()
return self.csAPI.initialize()
def execute( self ):
self.log.info( "Start Execution" )
result = getProxyInfo()
if not result[ 'OK' ]:
return result
infoDict = result[ 'Value' ]
self.log.info( formatProxyInfoAsString( infoDict ) )
#Get a "fresh" copy of the CS data
result = self.csAPI.downloadCSData()
if not result[ 'OK' ]:
self.log.warn( "Could not download a fresh copy of the CS data", result[ 'Message' ] )
self.__lookForCE()
self.__infoFromCE()
self.log.info( "End Execution" )
return S_OK()
def __checkAlternativeBDIISite( self, fun, *args ):
if self.alternativeBDIIs:
self.log.warn( "Trying to use alternative bdii sites" )
for site in self.alternativeBDIIs :
self.log.info( "Trying to contact alternative bdii ", site )
if len( args ) == 1 :
result = fun( args[0], host = site )
elif len( args ) == 2 :
result = fun( args[0], vo = args[1], host = site )
if not result['OK'] :
self.log.error ( "Problem contacting alternative bddii", result['Message'] )
elif result['OK'] :
return result
self.log.warn( "Also checking alternative BDII sites failed" )
return result
def __lookForCE( self ):
knownces = self.am_getOption( 'BannedCEs', [] )
result = gConfig.getSections( '/Resources/Sites' )
if not result['OK']:
return
grids = result['Value']
for grid in grids:
result = gConfig.getSections( '/Resources/Sites/%s' % grid )
if not result['OK']:
return
sites = result['Value']
for site in sites:
opt = gConfig.getOptionsDict( '/Resources/Sites/%s/%s' % ( grid, site ) )['Value']
ces = List.fromChar( opt.get( 'CE', '' ) )
knownces += ces
response = ldapCEState( '', vo = self.voName )
if not response['OK']:
self.log.error( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, '', self.voName )
return response
newces = {}
for queue in response['Value']:
try:
queuename = queue['GlueCEUniqueID']
except:
continue
cename = queuename.split( ":" )[0]
if not cename in knownces:
newces[cename] = None
self.log.debug( "newce", cename )
body = ""
possibleNewSites = []
for ce in newces.iterkeys():
response = ldapCluster( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCluster, ce )
continue
clusters = response['Value']
if len( clusters ) != 1:
self.log.warn( "Error in cluster length", " CE %s Length %d" % ( ce, len( clusters ) ) )
if len( clusters ) == 0:
continue
cluster = clusters[0]
fkey = cluster.get( 'GlueForeignKey', [] )
if type( fkey ) == type( '' ):
fkey = [fkey]
nameBDII = None
for entry in fkey:
if entry.count( 'GlueSiteUniqueID' ):
nameBDII = entry.split( '=' )[1]
break
if not nameBDII:
continue
cestring = "CE: %s, GOCDB Name: %s" % ( ce, nameBDII )
self.log.info( cestring )
response = ldapCE( ce )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
ceinfos = response['Value']
if len( ceinfos ):
ceinfo = ceinfos[0]
systemName = ceinfo.get( 'GlueHostOperatingSystemName', 'Unknown' )
systemVersion = ceinfo.get( 'GlueHostOperatingSystemVersion', 'Unknown' )
systemRelease = ceinfo.get( 'GlueHostOperatingSystemRelease', 'Unknown' )
else:
systemName = "Unknown"
systemVersion = "Unknown"
systemRelease = "Unknown"
osstring = "SystemName: %s, SystemVersion: %s, SystemRelease: %s" % ( systemName, systemVersion, systemRelease )
self.log.info( osstring )
response = ldapCEState( ce, vo = self.voName )
if not response['OK']:
self.log.warn( "Error during BDII request", response['Message'] )
response = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
newcestring = "\n\n%s\n%s" % ( cestring, osstring )
usefull = False
cestates = response['Value']
for cestate in cestates:
queuename = cestate.get( 'GlueCEUniqueID', 'UnknownName' )
queuestatus = cestate.get( 'GlueCEStateStatus', 'UnknownStatus' )
queuestring = "%s %s" % ( queuename, queuestatus )
self.log.info( queuestring )
newcestring += "\n%s" % queuestring
if queuestatus.count( 'Production' ):
usefull = True
if usefull:
body += newcestring
possibleNewSites.append( 'dirac-admin-add-site DIRACSiteName %s %s' % ( nameBDII, ce ) )
if body:
body = "We are glad to inform You about new CE(s) possibly suitable for %s:\n" % self.voName + body
body += "\n\nTo suppress information about CE add its name to BannedCEs list."
for possibleNewSite in possibleNewSites:
body = "%s\n%s" % ( body, possibleNewSite )
self.log.info( body )
if self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return S_OK()
def __infoFromCE( self ):
sitesSection = cfgPath( 'Resources', 'Sites' )
result = gConfig.getSections( sitesSection )
if not result['OK']:
return
grids = result['Value']
changed = False
body = ""
for grid in grids:
gridSection = cfgPath( sitesSection, grid )
result = gConfig.getSections( gridSection )
if not result['OK']:
return
sites = result['Value']
for site in sites:
siteSection = cfgPath( gridSection, site )
opt = gConfig.getOptionsDict( siteSection )['Value']
name = opt.get( 'Name', '' )
if name:
coor = opt.get( 'Coordinates', 'Unknown' )
mail = opt.get( 'Mail', 'Unknown' )
result = ldapSite( name )
if not result['OK']:
self.log.warn( "BDII site %s: %s" % ( name, result['Message'] ) )
result = self.__checkAlternativeBDIISite( ldapSite, name )
if result['OK']:
bdiisites = result['Value']
if len( bdiisites ) == 0:
self.log.warn( name, "Error in bdii: leng = 0" )
else:
if not len( bdiisites ) == 1:
self.log.warn( name, "Warning in bdii: leng = %d" % len( bdiisites ) )
bdiisite = bdiisites[0]
try:
longitude = bdiisite['GlueSiteLongitude']
latitude = bdiisite['GlueSiteLatitude']
newcoor = "%s:%s" % ( longitude, latitude )
except:
self.log.warn( "Error in bdii coor" )
newcoor = "Unknown"
try:
newmail = bdiisite['GlueSiteSysAdminContact'].split( ":" )[-1].strip()
except:
self.log.warn( "Error in bdii mail" )
newmail = "Unknown"
self.log.debug( "%s %s %s" % ( name, newcoor, newmail ) )
if newcoor != coor:
self.log.info( "%s" % ( name ), "%s -> %s" % ( coor, newcoor ) )
if coor == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Coordinates' ), newcoor )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Coordinates' ), newcoor )
changed = True
if newmail != mail:
self.log.info( "%s" % ( name ), "%s -> %s" % ( mail, newmail ) )
if mail == 'Unknown':
self.csAPI.setOption( cfgPath( siteSection, 'Mail' ), newmail )
else:
self.csAPI.modifyValue( cfgPath( siteSection, 'Mail' ), newmail )
changed = True
celist = List.fromChar( opt.get( 'CE', '' ) )
if not celist:
self.log.warn( site, 'Empty site list' )
continue
# result = gConfig.getSections( cfgPath( siteSection,'CEs' )
# if not result['OK']:
# self.log.debug( "Section CEs:", result['Message'] )
for ce in celist:
ceSection = cfgPath( siteSection, 'CEs', ce )
result = gConfig.getOptionsDict( ceSection )
if not result['OK']:
self.log.debug( "Section CE", result['Message'] )
wnTmpDir = 'Unknown'
arch = 'Unknown'
os = 'Unknown'
si00 = 'Unknown'
pilot = 'Unknown'
cetype = 'Unknown'
else:
ceopt = result['Value']
wnTmpDir = ceopt.get( 'wnTmpDir', 'Unknown' )
arch = ceopt.get( 'architecture', 'Unknown' )
os = ceopt.get( 'OS', 'Unknown' )
si00 = ceopt.get( 'SI00', 'Unknown' )
pilot = ceopt.get( 'Pilot', 'Unknown' )
cetype = ceopt.get( 'CEType', 'Unknown' )
result = ldapCE( ce )
if not result['OK']:
self.log.warn( 'Error in bdii for %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCE, ce )
continue
try:
bdiice = result['Value'][0]
except:
self.log.warn( 'Error in bdii for %s' % ce, result )
bdiice = None
if bdiice:
try:
newwnTmpDir = bdiice['GlueSubClusterWNTmpDir']
except:
newwnTmpDir = 'Unknown'
if wnTmpDir != newwnTmpDir and newwnTmpDir != 'Unknown':
section = cfgPath( ceSection, 'wnTmpDir' )
self.log.info( section, " -> ".join( ( wnTmpDir, newwnTmpDir ) ) )
if wnTmpDir == 'Unknown':
self.csAPI.setOption( section, newwnTmpDir )
else:
self.csAPI.modifyValue( section, newwnTmpDir )
changed = True
try:
newarch = bdiice['GlueHostArchitecturePlatformType']
except:
newarch = 'Unknown'
if arch != newarch and newarch != 'Unknown':
section = cfgPath( ceSection, 'architecture' )
self.log.info( section, " -> ".join( ( arch, newarch ) ) )
if arch == 'Unknown':
self.csAPI.setOption( section, newarch )
else:
self.csAPI.modifyValue( section, newarch )
changed = True
try:
newos = '_'.join( ( bdiice['GlueHostOperatingSystemName'],
bdiice['GlueHostOperatingSystemVersion'],
bdiice['GlueHostOperatingSystemRelease'] ) )
except:
newos = 'Unknown'
if os != newos and newos != 'Unknown':
section = cfgPath( ceSection, 'OS' )
self.log.info( section, " -> ".join( ( os, newos ) ) )
if os == 'Unknown':
self.csAPI.setOption( section, newos )
else:
self.csAPI.modifyValue( section, newos )
changed = True
body = body + "OS was changed %s -> %s for %s at %s\n" % ( os, newos, ce, site )
try:
newsi00 = bdiice['GlueHostBenchmarkSI00']
except:
newsi00 = 'Unknown'
if si00 != newsi00 and newsi00 != 'Unknown':
section = cfgPath( ceSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
try:
rte = bdiice['GlueHostApplicationSoftwareRunTimeEnvironment']
if self.voName.lower() == 'lhcb':
if 'VO-lhcb-pilot' in rte:
newpilot = 'True'
else:
newpilot = 'False'
else:
newpilot = 'Unknown'
except:
newpilot = 'Unknown'
if pilot != newpilot and newpilot != 'Unknown':
section = cfgPath( ceSection, 'Pilot' )
self.log.info( section, " -> ".join( ( pilot, newpilot ) ) )
if pilot == 'Unknown':
self.csAPI.setOption( section, newpilot )
else:
self.csAPI.modifyValue( section, newpilot )
changed = True
result = ldapCEState( ce, vo = self.voName ) #getBDIICEVOView
if not result['OK']:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Message'] )
result = self.__checkAlternativeBDIISite( ldapCEState, ce, self.voName )
continue
try:
queues = result['Value']
except:
self.log.warn( 'Error in bdii for queue %s' % ce, result['Massage'] )
continue
newcetype = 'Unknown'
for queue in queues:
try:
queuetype = queue['GlueCEImplementationName']
except:
queuetype = 'Unknown'
if newcetype == 'Unknown':
newcetype = queuetype
else:
if queuetype != newcetype:
self.log.warn( 'Error in bdii for ce %s ' % ce, 'different cetypes %s %s' % ( newcetype, queuetype ) )
if newcetype=='ARC-CE':
newcetype = 'ARC'
if cetype != newcetype and newcetype != 'Unknown':
section = cfgPath( ceSection, 'CEType' )
self.log.info( section, " -> ".join( ( cetype, newcetype ) ) )
if cetype == 'Unknown':
self.csAPI.setOption( section, newcetype )
else:
self.csAPI.modifyValue( section, newcetype )
changed = True
for queue in queues:
try:
queueName = queue['GlueCEUniqueID'].split( '/' )[-1]
except:
self.log.warn( 'error in queuename ', queue )
continue
try:
newmaxCPUTime = queue['GlueCEPolicyMaxCPUTime']
except:
newmaxCPUTime = None
newsi00 = None
try:
caps = queue['GlueCECapability']
if type( caps ) == type( '' ):
caps = [caps]
for cap in caps:
if cap.count( 'CPUScalingReferenceSI00' ):
newsi00 = cap.split( '=' )[-1]
except:
newsi00 = None
queueSection = cfgPath( ceSection, 'Queues', queueName )
result = gConfig.getOptionsDict( queueSection )
if not result['OK']:
self.log.warn( "Section Queues", result['Message'] )
maxCPUTime = 'Unknown'
si00 = 'Unknown'
else:
queueopt = result['Value']
maxCPUTime = queueopt.get( 'maxCPUTime', 'Unknown' )
si00 = queueopt.get( 'SI00', 'Unknown' )
if newmaxCPUTime and ( maxCPUTime != newmaxCPUTime ):
section = cfgPath( queueSection, 'maxCPUTime' )
self.log.info( section, " -> ".join( ( maxCPUTime, newmaxCPUTime ) ) )
if maxCPUTime == 'Unknown':
self.csAPI.setOption( section, newmaxCPUTime )
else:
self.csAPI.modifyValue( section, newmaxCPUTime )
changed = True
if newsi00 and ( si00 != newsi00 ):
section = cfgPath( queueSection, 'SI00' )
self.log.info( section, " -> ".join( ( si00, newsi00 ) ) )
if si00 == 'Unknown':
self.csAPI.setOption( section, newsi00 )
else:
self.csAPI.modifyValue( section, newsi00 )
changed = True
if changed:
self.log.info( body )
if body and self.addressTo and self.addressFrom:
notification = NotificationClient()
result = notification.sendMail( self.addressTo, self.subject, body, self.addressFrom, localAttempt = False )
return self.csAPI.commit()
else:
self.log.info( "No changes found" )
return S_OK()
class DDSimTarMaker( object ):
""" create a tarball of the DDSim release """
def __init__( self ):
self.detmodels = {}
self.lcgeo_env="lcgeo_DIR"
self.ddhep_env="DD4HEP"
self.softSec = "/Operations/Defaults/AvailableTarBalls"
self.version = ''
self.platform = 'x86_64-slc5-gcc43-opt'
self.comment = ""
self.name = "ddsim"
self.csapi = None
self.tarBallName = None
self.md5sum = None
def copyDetectorModels( self, basePath, folder, targetFolder ):
"""copy the compact folders to the targetFolder """
for root,dirs,_files in os.walk( os.path.join(basePath, folder) ):
for direct in dirs:
if root.endswith("compact"):
## the main xml file must have the same name as the folder, ILD and CLIC follow this convention already
xmlPath = os.path.join( root, direct, direct+".xml")
if os.path.exists( xmlPath ):
self.detmodels[direct] = "detectors/"+direct+"/"+direct+".xml"
copyFolder( os.path.join(root, direct), targetFolder )
def createTarBall( self, folder ):
"""create a tar ball from the folder
tar zcf $TARBALLNAME $LIBFOLDER/*
"""
##Create the Tarball
if os.path.exists(self.tarBallName):
os.remove(self.tarBallName)
gLogger.notice("Creating Tarball...")
myappTar = tarfile.open(self.tarBallName, "w:gz")
myappTar.add(folder, self.tarBallName[:-4])
myappTar.close()
self.md5sum = md5.md5(open( self.tarBallName, 'r' ).read()).hexdigest()
gLogger.notice("...Done")
return S_OK( "Created Tarball")
def parseArgs( self ):
""" parse the command line arguments"""
if len(sys.argv) != 3:
raise RuntimeError( "Wrong number of arguments in call: '%s'" % " ".join(sys.argv) )
self.name = sys.argv[1]
self.version = sys.argv[2]
self.tarBallName = "%s%s.tgz" % (self.name, self.version)
def checkEnvironment( self ):
""" check if dd4hep and lcgeo are in the environment """
for var in [ self.ddhep_env, self.lcgeo_env , 'ROOTSYS' ]:
if var not in os.environ:
raise RuntimeError( "%s is not set" % var )
return os.environ[self.ddhep_env], os.environ[self.lcgeo_env], os.environ['ROOTSYS']
def createCSEntry( self ):
"""add the entries for this version into the Configuration System
<version>
{
TarBall = ddsim<version>.tgz
AdditionalEnvVar
{
ROOTSYS = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/root/5.34.30
G4INSTALL = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01
G4DATA = /cvmfs/ilc.desy.de/sw/x86_64_gcc44_sl6/geant4/10.01/share/Geant4-10.1.0/data
}
Overwrite = True
}
Operations/DDSimDetectorModels/<Version>
{
CLIC_o2_v03 = detectors/CLIC_o2_v03/CLIC_o2_v03.xml
...
}
"""
#FIXME: Get root and geant4 location from environment, make sure it is cvmfs
csParameter = { "TarBall": self.tarBallName,
"AdditionalEnvVar": {
"ROOTSYS" : os.environ.get("ROOTSYS"),
"G4INSTALL" : os.environ.get("G4INSTALL"),
"G4DATA" : os.environ.get("G4DATA"),
},
"Md5Sum": self.md5sum,
}
pars = dict( platform=self.platform,
name="ddsim",
version=self.version
)
csPath = os.path.join( self.softSec , "%(platform)s/%(name)s/%(version)s/" % pars )
pprint(csParameter)
result = self.insertCSSection( csPath, csParameter )
csPathModels = "Operations/Defaults/DDSimDetectorModels"
csModels = { self.version : self.detmodels }
pprint(csModels)
result = self.insertCSSection( csPathModels, csModels )
if self.csapi is not None:
resProxy = checkOrGetGroupProxy( "diracAdmin" )
if not resProxy['OK']:
gLogger.error( "Failed to get AdminProxy", resProxy['Message'] )
raise RuntimeError( "Failed to get diracAdminProxy" )
self.csapi.commit()
if not result['OK']:
gLogger.error( "Failed to create CS Section", result['Message'] )
raise RuntimeError( "Failed to create CS Section" )
def insertCSSection( self, path, pardict ):
""" insert a section and values (or subsections) into the CS
:param str path: full path of the new section
:param str pardict: dictionary of key values in the new section, values can also be dictionaries
:return: S_OK(), S_ERROR()
"""
from DIRAC.ConfigurationSystem.Client.CSAPI import CSAPI
if self.csapi is None:
self.csapi = CSAPI()
for key, value in pardict.iteritems():
newSectionPath = os.path.join(path,key)
gLogger.debug( "Adding to cs %s : %s " % ( newSectionPath, value ) )
self.csapi.createSection( path )
if isinstance( value, dict ):
res = self.insertCSSection( newSectionPath, value )
else:
res = self.csapi.setOption( newSectionPath, value )
if not res['OK']:
return res
else:
gLogger.notice( "Added to CS: %s " % res['Value'] )
return S_OK("Added all things to cs")
def createDDSimTarBall( self ):
""" do everything to create the DDSim tarball"""
self.parseArgs()
ddBase, lcgeoBase, _rootsys = self.checkEnvironment()
realTargetFolder = os.path.join( os.getcwd(), self.name+self.version )
targetFolder = os.path.join( os.getcwd(), "temp", self.name+self.version )
for folder in (targetFolder, targetFolder+"/lib"):
try:
os.makedirs( folder )
except OSError:
pass
libraries = set()
rootmaps = set()
dd4hepLibPath = getLibraryPath( ddBase )
lcgeoPath = getLibraryPath( lcgeoBase )
self.copyDetectorModels( lcgeoBase, "CLIC" , targetFolder+"/detectors" )
self.copyDetectorModels( lcgeoBase, "ILD" , targetFolder+"/detectors" )
copyFolder( ddBase+"/DDDetectors/compact", realTargetFolder.rstrip("/")+"/DDDetectors")
libraries.update( getFiles( dd4hepLibPath, ".so") )
libraries.update( getFiles( lcgeoPath, ".so" ) )
rootmaps.update( getFiles( dd4hepLibPath, ".rootmap") )
rootmaps.update( getFiles( lcgeoPath, ".rootmap" ) )
pprint( libraries )
pprint( rootmaps )
allLibs = set()
for lib in libraries:
allLibs.update( getDependentLibraries(lib) )
### remote root and geant4 libraries, we pick them up from
allLibs = set( [ lib for lib in allLibs if not ( "/geant4/" in lib.lower() or "/root/" in lib.lower()) ] )
print allLibs
copyLibraries( libraries, targetFolder+"/lib" )
copyLibraries( allLibs, targetFolder+"/lib" )
copyLibraries( rootmaps, targetFolder+"/lib" )
getPythonStuff( ddBase+"/python" , targetFolder+"/lib/")
getPythonStuff( lcgeoBase+"/lib/python", targetFolder+"/lib/" )
getPythonStuff( lcgeoBase+"/bin/ddsim", targetFolder+"/bin/" )
##Should get this from CVMFS
#getRootStuff( rootsys, targetFolder+"/ROOT" )
copyFolder( targetFolder+"/", realTargetFolder.rstrip("/") )
killRPath( realTargetFolder )
resolveLinks( realTargetFolder+"/lib" )
removeSystemLibraries( realTargetFolder+"/lib" )
#removeSystemLibraries( realTargetFolder+"/ROOT/lib" )
print self.detmodels
self.createTarBall( realTargetFolder )
self.createCSEntry()
def checkUnusedSEs():
global vo, dry
result = getGridSRMs( vo, unUsed = True )
if not result['OK']:
gLogger.error( 'Failed to look up SRMs in BDII', result['Message'] )
siteSRMDict = result['Value']
# Evaluate VOs
result = getVOs()
if result['OK']:
csVOs = set( result['Value'] )
else:
csVOs = set( [vo] )
changeSetFull = set()
for site in siteSRMDict:
for gridSE in siteSRMDict[site]:
changeSet = set()
seDict = siteSRMDict[site][gridSE]['SE']
srmDict = siteSRMDict[site][gridSE]['SRM']
# Check the SRM version
version = srmDict.get( 'GlueServiceVersion', '' )
if not ( version and version.startswith( '2' ) ):
gLogger.debug( 'Skipping SRM service with version %s' % version )
continue
result = getDIRACSiteName( site )
if not result['OK']:
gLogger.notice( 'Unused se %s is detected at unused site %s' % ( gridSE, site ) )
gLogger.notice( 'Consider adding site %s to the DIRAC CS' % site )
continue
diracSites = result['Value']
yn = raw_input( '\nDo you want to add new SRM SE %s at site(s) %s ? default yes [yes|no]: ' % ( gridSE, str( diracSites ) ) )
if not yn or yn.lower().startswith( 'y' ):
if len( diracSites ) > 1:
prompt = 'Which DIRAC site the new SE should be attached to ?'
for i, s in enumerate( diracSites ):
prompt += '\n[%d] %s' % ( i, s )
prompt += '\nEnter your choice number: '
inp = raw_input( prompt )
try:
ind = int( inp )
except:
gLogger.notice( 'Can not interpret your choice: %s, try again later' % inp )
continue
diracSite = diracSites[ind]
else:
diracSite = diracSites[0]
domain, siteName, country = diracSite.split( '.' )
recName = '%s-disk' % siteName
inp = raw_input( 'Give a DIRAC name to the grid SE %s, default %s : ' % ( gridSE, recName ) )
diracSEName = inp
if not inp:
diracSEName = recName
gLogger.notice( 'Adding new SE %s at site %s' % ( diracSEName, diracSite ) )
seSection = cfgPath( '/Resources/StorageElements', diracSEName )
changeSet.add( ( seSection, 'BackendType', seDict.get( 'GlueSEImplementationName', 'Unknown' ) ) )
changeSet.add( ( seSection, 'Description', seDict.get( 'GlueSEName', 'Unknown' ) ) )
bdiiVOs = set( [ re.sub( '^VO:', '', rule ) for rule in srmDict.get( 'GlueServiceAccessControlBaseRule', [] ) ] )
seVOs = csVOs.intersection( bdiiVOs )
changeSet.add( ( seSection, 'VO', ','.join( seVOs ) ) )
accessSection = cfgPath( seSection, 'AccessProtocol.1' )
changeSet.add( ( accessSection, 'Protocol', 'srm' ) )
changeSet.add( ( accessSection, 'ProtocolName', 'SRM2' ) )
endPoint = srmDict.get( 'GlueServiceEndpoint', '' )
result = pfnparse( endPoint )
if not result['OK']:
gLogger.error( 'Can not get the SRM service end point. Skipping ...' )
continue
host = result['Value']['Host']
port = result['Value']['Port']
changeSet.add( ( accessSection, 'Host', host ) )
changeSet.add( ( accessSection, 'Port', port ) )
changeSet.add( ( accessSection, 'Access', 'remote' ) )
voPathSection = cfgPath( accessSection, 'VOPath' )
if 'VOPath' in seDict:
path = seDict['VOPath']
voFromPath = os.path.basename( path )
if voFromPath != diracVO:
gLogger.notice( '\n!!! Warning: non-conventional VO path: %s\n' % path )
changeSet.add( ( voPathSection, diracVO, path ) )
path = os.path.dirname( path )
else:
# Try to guess the Path
domain = '.'.join( host.split( '.' )[-2:] )
path = '/dpm/%s/home' % domain
changeSet.add( ( accessSection, 'Path', path ) )
changeSet.add( ( accessSection, 'SpaceToken', '' ) )
changeSet.add( ( accessSection, 'WSUrl', '/srm/managerv2?SFN=' ) )
gLogger.notice( 'SE %s will be added with the following parameters' % diracSEName )
changeList = list( changeSet )
changeList.sort()
for entry in changeList:
gLogger.notice( entry )
yn = raw_input( 'Do you want to add new SE %s ? default yes [yes|no]: ' % diracSEName )
if not yn or yn.lower().startswith( 'y' ):
changeSetFull = changeSetFull.union( changeSet )
if dry:
if changeSetFull:
gLogger.notice( 'Skipping commit of the new SE data in a dry run' )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
if changeSetFull:
csAPI = CSAPI()
csAPI.initialize()
result = csAPI.downloadCSData()
if not result['OK']:
gLogger.error( 'Failed to initialize CSAPI object', result['Message'] )
DIRACExit( -1 )
changeList = list( changeSetFull )
changeList.sort()
for section, option, value in changeList:
csAPI.setOption( cfgPath( section, option ), value )
yn = raw_input( 'New SE data is accumulated\n Do you want to commit changes to CS ? default yes [yes|no]: ' )
if not yn or yn.lower().startswith( 'y' ):
result = csAPI.commit()
if not result['OK']:
gLogger.error( "Error while commit to CS", result['Message'] )
else:
gLogger.notice( "Successfully committed %d changes to CS" % len( changeSetFull ) )
else:
gLogger.notice( "No new SE to be added" )
return S_OK()
