def initializeHandler(cls, svcInfoDict):
"""WMS AdministratorService initialization"""
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobDB", "JobDB")
if not result["OK"]:
return result
cls.jobDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR(f"Can't connect to DB: {excp!r}")
cls.elasticJobParametersDB = None
useESForJobParametersFlag = Operations().getValue(
"/Services/JobMonitoring/useESForJobParametersFlag", False)
if useESForJobParametersFlag:
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.ElasticJobParametersDB",
"ElasticJobParametersDB")
if not result["OK"]:
return result
cls.elasticJobParametersDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR(f"Can't connect to DB: {excp!r}")
cls.pilotManager = PilotManagerClient()
return S_OK()
def initialize(self):
"""Standard initialize."""
res = ObjectLoader().loadObject(
"DIRAC.ResourceStatusSystem.Client.ResourceManagementClient")
if not res["OK"]:
self.log.error(
"Failed to load ResourceManagementClient class: %s" %
res["Message"])
return res
rmClass = res["Value"]
res = ObjectLoader().loadObject(
"DIRAC.ResourceStatusSystem.Client.ResourceStatusClient")
if not res["OK"]:
self.log.error("Failed to load ResourceStatusClient class: %s" %
res["Message"])
return res
rsClass = res["Value"]
self.rsClient = rsClass()
self.clients["ResourceStatusClient"] = rsClass()
self.clients["ResourceManagementClient"] = rmClass()
maxNumberOfThreads = self.am_getOption("maxNumberOfThreads", 15)
self.log.info("Multithreaded with %d threads" % maxNumberOfThreads)
self.threadPoolExecutor = concurrent.futures.ThreadPoolExecutor(
max_workers=maxNumberOfThreads)
return S_OK()
def initialize(self):
""" Standard initialize.
"""
maxNumberOfThreads = self.am_getOption('maxNumberOfThreads',
self.__maxNumberOfThreads)
self.threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.SiteStatus', 'SiteStatus')
if not res['OK']:
self.log.error('Failed to load SiteStatus class: %s' %
res['Message'])
return res
siteStatusClass = res['Value']
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error(
'Failed to load ResourceManagementClient class: %s' %
res['Message'])
return res
rmClass = res['Value']
self.siteClient = siteStatusClass()
self.clients['SiteStatus'] = siteStatusClass()
self.clients['ResourceManagementClient'] = rmClass()
return S_OK()
def initializeHandler(cls, svcInfoDict):
"""
Determines the switching of ElasticSearch and MySQL backends
"""
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobDB", "JobDB")
if not result["OK"]:
return result
cls.jobDB = result["Value"]()
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
if not result["OK"]:
return result
cls.jobLoggingDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.elasticJobParametersDB = None
useESForJobParametersFlag = Operations().getValue(
"/Services/JobMonitoring/useESForJobParametersFlag", False)
if useESForJobParametersFlag:
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.ElasticJobParametersDB",
"ElasticJobParametersDB")
if not result["OK"]:
return result
cls.elasticJobParametersDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
return S_OK()
def initializeHandler(cls, serviceInfoDict):
"""Initialization of DB objects"""
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
if not result["OK"]:
return result
cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.pilotsLoggingDB = None
enablePilotsLogging = Operations().getValue(
"/Services/JobMonitoring/usePilotsLoggingFlag", False)
if enablePilotsLogging:
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.PilotsLoggingDB",
"PilotsLoggingDB")
if not result["OK"]:
return result
cls.pilotsLoggingDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
return S_OK()
def initializeHandler(cls, serviceInfoDict):
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobDB", "JobDB")
if not result["OK"]:
return result
cls.jobDB = result["Value"](parentLogger=cls.log)
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
if not result["OK"]:
return result
cls.jobLoggingDB = result["Value"](parentLogger=cls.log)
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
if not result["OK"]:
return result
cls.taskQueueDB = result["Value"](parentLogger=cls.log)
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
if not result["OK"]:
return result
cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.limiter = Limiter(jobDB=cls.jobDB)
return S_OK()
def initializeHandler(cls, serviceInfoDict):
try:
result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobDB", "JobDB")
if not result["OK"]:
return result
cls.jobDB = result["Value"]()
result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
if not result["OK"]:
return result
cls.jobLoggingDB = result["Value"]()
result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
if not result["OK"]:
return result
cls.taskQueueDB = result["Value"]()
result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
if not result["OK"]:
return result
cls.pilotAgentsDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.limiter = Limiter(jobDB=cls.jobDB)
gMonitor.registerActivity("matchTime", "Job matching time", "Matching", "secs", gMonitor.OP_MEAN, 300)
gMonitor.registerActivity("matchesDone", "Job Match Request", "Matching", "matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity("matchesOK", "Matched jobs", "Matching", "matches", gMonitor.OP_RATE, 300)
gMonitor.registerActivity("numTQs", "Number of Task Queues", "Matching", "tqsk queues", gMonitor.OP_MEAN, 300)
return S_OK()
def initialize(self):
""" Define the commands to be executed, and instantiate the clients that will be used.
"""
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' % res['Message'])
return res
rsClass = res['Value']
res = ObjectLoader().loadObject('DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error('Failed to load ResourceManagementClient class: %s' % res['Message'])
return res
rmClass = res['Value']
self.commands['Downtime'] = [{'Downtime': {}}]
self.commands['GOCDBSync'] = [{'GOCDBSync': {}}]
self.commands['FreeDiskSpace'] = [{'FreeDiskSpace': {}}]
# PilotsCommand
# self.commands[ 'Pilots' ] = [
# { 'PilotsWMS' : { 'element' : 'Site', 'siteName' : None } },
# { 'PilotsWMS' : { 'element' : 'Resource', 'siteName' : None } }
# ]
# FIXME: do not forget about hourly vs Always ...etc
# AccountingCacheCommand
# self.commands[ 'AccountingCache' ] = [
# {'SuccessfullJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'FailedJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# {'SuccessfullPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsBySiteSplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'SuccessfullPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'FailedPilotsByCESplitted' :{'hours' :24, 'plotType' :'Pilot' }},
# {'RunningJobsBySiteSplitted' :{'hours' :24, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :168, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :720, 'plotType' :'Job' }},
# # {'RunningJobsBySiteSplitted' :{'hours' :8760, 'plotType' :'Job' }},
# ]
# VOBOXAvailability
# self.commands[ 'VOBOXAvailability' ] = [
# { 'VOBOXAvailability' : {} }
#
# Reuse clients for the commands
self.clients['GOCDBClient'] = GOCDBClient()
self.clients['ReportsClient'] = ReportsClient()
self.clients['ResourceStatusClient'] = rsClass()
self.clients['ResourceManagementClient'] = rmClass()
self.clients['WMSAdministrator'] = WMSAdministratorClient()
self.clients['Pilots'] = PilotManagerClient()
self.cCaller = CommandCaller
return S_OK()
def __init__(self, clients=dict()):
""" Constructor
examples:
>>> pep = PEP()
>>> pep1 = PEP( { 'ResourceStatusClient' : ResourceStatusClient() } )
>>> pep2 = PEP( { 'ResourceStatusClient' : ResourceStatusClient(), 'ClientY' : None } )
:Parameters:
**clients** - [ None, `dict` ]
dictionary with clients to be used in the commands issued by the policies.
If not defined, the commands will import them. It is a measure to avoid
opening the same connection every time a policy is evaluated.
"""
self.clients = dict(clients)
# Creating the client in the PEP is a convenience for the PDP, that uses internally the RSS clients
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceStatusClient',
'ResourceStatusClient')
if not res['OK']:
self.log.error('Failed to load ResourceStatusClient class: %s' %
res['Message'])
raise ImportError(res['Message'])
rsClass = res['Value']
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.ResourceManagementClient',
'ResourceManagementClient')
if not res['OK']:
self.log.error(
'Failed to load ResourceManagementClient class: %s' %
res['Message'])
raise ImportError(res['Message'])
rmClass = res['Value']
res = ObjectLoader().loadObject(
'DIRAC.ResourceStatusSystem.Client.SiteStatus', 'SiteStatus')
if not res['OK']:
self.log.error('Failed to load SiteStatus class: %s' %
res['Message'])
raise ImportError(res['Message'])
ssClass = res['Value']
if 'ResourceStatusClient' not in clients:
self.clients['ResourceStatusClient'] = rsClass()
if 'ResourceManagementClient' not in clients:
self.clients['ResourceManagementClient'] = rmClass()
if 'SiteStatus' not in clients:
self.clients['SiteStatus'] = ssClass()
# Pass to the PDP the clients that are going to be used on the Commands
self.pdp = PDP(self.clients)
self.log = gLogger
def initializeHandler(cls, serviceInfoDict):
"""Initialization of DB objects and OptimizationMind"""
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobDB", "JobDB")
if not result["OK"]:
return result
cls.jobDB = result["Value"](parentLogger=cls.log)
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
if not result["OK"]:
return result
cls.jobLoggingDB = result["Value"](parentLogger=cls.log)
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
if not result["OK"]:
return result
cls.taskQueueDB = result["Value"](parentLogger=cls.log)
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.PilotAgentsDB", "PilotAgentsDB")
if not result["OK"]:
return result
cls.pilotAgentsDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.pilotsLoggingDB = None
enablePilotsLogging = Operations().getValue(
"/Services/JobMonitoring/usePilotsLoggingFlag", False)
if enablePilotsLogging:
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.PilotsLoggingDB",
"PilotsLoggingDB")
if not result["OK"]:
return result
cls.pilotsLoggingDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.msgClient = MessageClient("WorkloadManagement/OptimizationMind")
result = cls.msgClient.connect(JobManager=True)
if not result["OK"]:
cls.log.warn("Cannot connect to OptimizationMind!",
result["Message"])
return S_OK()
def registerBackend(self, desiredBackend, backendOptions=None):
"""
Attach a backend to the Logging object.
Convert backend name to backend class name to a Backend object and add it to the Logging object
:params desiredBackend: a name attaching to a backend type.
list of the possible values: ['stdout', 'stderr', 'file', 'server']
:params backendOptions: dictionary of different backend options.
example: FileName='/tmp/log.txt'
"""
# import ObjectLoader here to avoid a dependancy loop
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
objLoader = ObjectLoader()
# Remove white space and capitalize the first letter
desiredBackend = desiredBackend.strip()
desiredBackend = desiredBackend[0].upper() + desiredBackend[1:]
# lock to avoid problem in ObjectLoader which is a singleton not thread-safe
self._lockObjectLoader.acquire()
try:
# load the Backend class
_class = objLoader.loadObject(
'DIRAC.Resources.LogBackends.%sBackend' % desiredBackend)
finally:
self._lockObjectLoader.release()
if _class['OK']:
# add the backend instance to the Logging
self._addBackend(_class['Value'](), backendOptions)
self._generateBackendFormat()
else:
self._generateBackendFormat()
self.warn("%s is not a valid backend name." % desiredBackend)
def initialize(self):
"""Sets default parameters and creates CE instance"""
super(PushJobAgent, self).initialize()
result = self._initializeComputingElement("Pool")
if not result["OK"]:
return result
# on-the fly imports
ol = ObjectLoader()
res = ol.loadModule("ConfigurationSystem.Client.Helpers.Resources")
if not res["OK"]:
sys.exit(res["Message"])
self.resourcesModule = res["Value"]
self.opsHelper = Operations()
# Disable Watchdog: we don't need it as pre/post processing occurs locally
setup = gConfig.getValue("/DIRAC/Setup", "")
if not setup:
return S_ERROR("Cannot get the DIRAC Setup value")
wms_instance = getSystemInstance("WorkloadManagement")
if not wms_instance:
return S_ERROR("Cannot get the WorkloadManagement system instance")
section = "/Systems/WorkloadManagement/%s/JobWrapper" % wms_instance
self._updateConfiguration("CheckWallClockFlag", 0, path=section)
self._updateConfiguration("CheckDiskSpaceFlag", 0, path=section)
self._updateConfiguration("CheckLoadAvgFlag", 0, path=section)
self._updateConfiguration("CheckCPUConsumedFlag", 0, path=section)
self._updateConfiguration("CheckCPULimitFlag", 0, path=section)
self._updateConfiguration("CheckMemoryLimitFlag", 0, path=section)
self._updateConfiguration("CheckTimeLeftFlag", 0, path=section)
return S_OK()
def initializeHandler(cls, serviceInfo):
try:
result = ObjectLoader().loadObject(
"MonitoringSystem.DB.MonitoringDB", "MonitoringDB")
if not result["OK"]:
return result
cls.__db = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
reportSection = serviceInfo["serviceSectionPath"]
dataPath = gConfig.getValue("%s/DataLocation" % reportSection,
"data/monitoringPlots")
gLogger.info("Data will be written into %s" % dataPath)
mkDir(dataPath)
try:
testFile = "%s/moni.plot.test" % dataPath
with open(testFile, "w") as _:
os.unlink(testFile)
except IOError as err:
gLogger.fatal("Can't write to %s" % dataPath, err)
return S_ERROR("Data location is not writable: %s" % repr(err))
gDataCache.setGraphsLocation(dataPath)
return S_OK()
def initialize(self, jobDB=None, logDB=None):
"""Initialization of the Optimizer Agent."""
self.jobDB = JobDB() if jobDB is None else jobDB
if not self.jobDB.isValid():
dExit(1)
useESForJobParametersFlag = Operations().getValue(
"/Services/JobMonitoring/useESForJobParametersFlag", False)
if useESForJobParametersFlag:
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.ElasticJobParametersDB",
"ElasticJobParametersDB")
if not result["OK"]:
return result
self.elasticJobParametersDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
self.logDB = JobLoggingDB() if logDB is None else logDB
optimizerName = self.am_getModuleParam("agentName")
if optimizerName.endswith("Agent"):
optimizerName = optimizerName[:-len("Agent")]
self.am_setModuleParam("optimizerName", optimizerName)
self.startingMinorStatus = self.am_getModuleParam("optimizerName")
self.failedStatus = self.am_getOption("FailedJobStatus",
JobStatus.FAILED)
self.am_setOption("PollingTime", 30)
return self.initializeOptimizer()
def initialize(self):
"""Standard initialize method"""
res = TaskManagerAgentBase.initialize(self)
if not res["OK"]:
return res
objLoader = ObjectLoader()
_class = objLoader.loadObject(
"TransformationSystem.Client.RequestTasks", "RequestTasks")
if not _class["OK"]:
raise Exception(_class["Message"])
self.requestTasksCls = _class["Value"]
# clients
self.taskManager = self.requestTasksCls(transClient=self.transClient)
agentTSTypes = self.am_getOption("TransType", [])
if agentTSTypes:
self.transType = agentTSTypes
else:
self.transType = Operations().getValue(
"Transformations/DataManipulation", ["Replication", "Removal"])
return S_OK()
def __generateStorageObject(self, storageName, pluginName, parameters, hideExceptions=False):
"""
Generate a Storage Element from parameters collected
:param storageName: is the storage section to check in the CS
:param pluginName: name of the plugin used. Example: GFAL2_XROOT, GFAL2_SRM2...
:param parameters: dictionary of protocol details.
"""
storageType = pluginName
if self.proxy:
storageType = 'Proxy'
objectLoader = ObjectLoader()
result = objectLoader.loadObject('Resources.Storage.%sStorage' % storageType, storageType + 'Storage',
hideExceptions=hideExceptions)
if not result['OK']:
gLogger.error('Failed to load storage object: %s' % result['Message'])
return result
storageClass = result['Value']
try:
storage = storageClass(storageName, parameters)
except Exception as x:
errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s: %s" % (storageName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
return S_OK(storage)
def initializeHandler(cls, serviceInfoDict):
useMyProxy = cls.srv_getCSOption("UseMyProxy", False)
try:
result = ObjectLoader().loadObject('FrameworkSystem.DB.ProxyDB',
'ProxyDB')
if not result['OK']:
gLogger.error('Failed to load ProxyDB class: %s' %
result['Message'])
return result
dbClass = result['Value']
cls.__proxyDB = dbClass(useMyProxy=useMyProxy)
except RuntimeError as excp:
return S_ERROR("Can't connect to ProxyDB: %s" % excp)
gThreadScheduler.addPeriodicTask(900,
cls.__proxyDB.purgeExpiredTokens,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(900,
cls.__proxyDB.purgeExpiredRequests,
elapsedTime=900)
gThreadScheduler.addPeriodicTask(21600, cls.__proxyDB.purgeLogs)
gThreadScheduler.addPeriodicTask(3600,
cls.__proxyDB.purgeExpiredProxies)
gLogger.info("MyProxy: %s\n MyProxy Server: %s" %
(useMyProxy, cls.__proxyDB.getMyProxyServer()))
return S_OK()
def __getCatalogClass(self, catalogType):
result = ObjectLoader().loadObject("Resources.Catalog.%sClient" %
catalogType)
if not result["OK"]:
gLogger.error("Failed to load catalog object",
"%s" % result["Message"])
return result
def __generateStorageObject(self,
storageName,
pluginName,
parameters,
hideExceptions=False):
storageType = pluginName
if self.proxy:
storageType = 'Proxy'
objectLoader = ObjectLoader()
result = objectLoader.loadObject('Resources.Storage.%sStorage' %
storageType,
storageType + 'Storage',
hideExceptions=hideExceptions)
if not result['OK']:
gLogger.error('Failed to load storage object: %s' %
result['Message'])
return result
storageClass = result['Value']
try:
storage = storageClass(storageName, parameters)
except Exception, x:
errStr = "StorageFactory._generateStorageObject: Failed to instantiate %s: %s" % (
storageName, x)
gLogger.exception(errStr)
return S_ERROR(errStr)
def initializeHandler(cls, serviceInfoDict):
"""Initialization of Pilots Logging service"""
cls.consumersSet = set()
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.PilotsLoggingDB",
"PilotsLoggingDB")
if not result["OK"]:
return result
cls.pilotsLoggingDB = result["Value"](parentLogger=cls.log)
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
queue = cls.srv_getCSOption("PilotsLoggingQueue")
# This is pretty awful hack. Somehow, for uknown reason, I cannot access CS with srv_getCSOption.
# The only way is using full CS path, so I'm using it as a backup solution.
if not queue:
queue = gConfig.getValue(serviceInfoDict["serviceSectionPath"] +
"/PilotsLoggingQueue")
result = createConsumer(queue, callback=cls.consumingCallback)
if result["OK"]:
cls.consumersSet.add(result["Value"])
else:
return result
return S_OK()
def createCatalog(self,
catalogName,
useProxy=False,
vo=None,
catalogConfig={}):
""" Create a file catalog object from its name and CS description
"""
if useProxy:
catalog = FileCatalogProxyClient(catalogName)
return S_OK(catalog)
# get the CS description first
catConfig = catalogConfig
if not catConfig:
if not vo:
result = getVOfromProxyGroup()
if not result['OK']:
return result
vo = result['Value']
reHelper = Resources(vo=vo)
result = reHelper.getCatalogOptionsDict(catalogName)
if not result['OK']:
return result
catConfig = result['Value']
catalogType = catConfig.get('CatalogType', catalogName)
catalogURL = catConfig.get('CatalogURL', '')
self.log.verbose('Creating %s client' % catalogName)
objectLoader = ObjectLoader()
result = objectLoader.loadObject(
'Resources.Catalog.%sClient' % catalogType, catalogType + 'Client')
if not result['OK']:
gLogger.error('Failed to load catalog object: %s' %
result['Message'])
return result
catalogClass = result['Value']
try:
if catalogType in ['LcgFileCatalogCombined', 'LcgFileCatalog']:
# The LFC special case
infoSys = catConfig.get('LcgGfalInfosys', '')
host = catConfig.get('MasterHost', '')
catalog = catalogClass(infoSys, host)
else:
if catalogURL:
catalog = catalogClass(url=catalogURL)
else:
catalog = catalogClass()
self.log.debug('Loaded module %sClient' % catalogType)
return S_OK(catalog)
except Exception, x:
errStr = "Failed to instantiate %s()" % (catalogType)
gLogger.exception(errStr, lException=x)
return S_ERROR(errStr)
def __getCorrectorClass(self, correctorName):
baseImport = "WorkloadManagementSystem.private.correctors"
fullCN = "%s.%sCorrector" % (baseImport, correctorName)
result = ObjectLoader().getObjects(baseImport, ".*Corrector", parentClass=BaseCorrector)
if not result["OK"]:
return result
data = result["Value"]
if fullCN not in data:
return S_ERROR("Can't find corrector %s" % fullCN)
return S_OK(data[fullCN])
def __loadCatalogComponent(self, componentType, componentName):
"""Create an object of a given catalog component"""
componentModule = "DataManagementSystem.DB.FileCatalogComponents.%s.%s" % (componentType, componentName)
result = ObjectLoader().loadObject(componentModule)
if not result["OK"]:
gLogger.error("Failed to load catalog component", "%s: %s" % (componentName, result["Message"]))
return result
componentClass = result["Value"]
component = componentClass(self)
return S_OK(component)
def initializeHandler(cls, serviceInfoDict):
"""Initialization of DB object"""
try:
result = ObjectLoader().loadObject("WorkloadManagementSystem.DB.SandboxMetadataDB", "SandboxMetadataDB")
if not result["OK"]:
return result
cls.sandboxDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
return S_OK()
def initializeHandler(cls, svcInfoDict):
"""initialize DBs"""
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobDB", "JobDB")
if not result["OK"]:
return result
cls.jobDB = result["Value"]()
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.JobLoggingDB", "JobLoggingDB")
if not result["OK"]:
return result
cls.jobLoggingDB = result["Value"]()
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.TaskQueueDB", "TaskQueueDB")
if not result["OK"]:
return result
cls.taskQueueDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.elasticJobParametersDB = None
useESForJobParametersFlag = Operations().getValue(
"/Services/JobMonitoring/useESForJobParametersFlag", False)
if useESForJobParametersFlag:
try:
result = ObjectLoader().loadObject(
"WorkloadManagementSystem.DB.ElasticJobParametersDB",
"ElasticJobParametersDB")
if not result["OK"]:
return result
cls.elasticJobParametersDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
cls.pilotManager = PilotManagerClient()
return S_OK()
def loadBatchSystem( self ):
""" Instantiate object representing the backend batch system
"""
self.batchSystem = self.ceParameters['BatchSystem']
objectLoader = ObjectLoader()
result = objectLoader.loadObject( 'Resources.Computing.BatchSystems.%s' % self.batchSystem, self.batchSystem )
if not result['OK']:
gLogger.error( 'Failed to load batch object: %s' % result['Message'] )
return result
batchClass = result['Value']
self.batchModuleFile = result['ModuleFile']
self.batch = batchClass()
self.log.info( "Batch system class from module: ", self.batchModuleFile )
def initializeHandler(cls, serviceInfoDict):
"""Initialization of DB object"""
try:
result = ObjectLoader().loadObject(
"ProductionSystem.DB.ProductionDB", "ProductionDB")
if not result["OK"]:
return result
cls.productionDB = result["Value"]()
except RuntimeError as excp:
return S_ERROR("Can't connect to DB: %s" % excp)
return S_OK()
def __init__(self, autoDiscovery=True):
"""
Initialization function, you can set autoDiscovery=False to prevent automatic
discovery of handler. If disabled you can use loadHandlersByServiceName() to
load your handlers or loadHandlerInHandlerManager()
:param autoDiscovery: (default True) Disable the automatic discovery,
can be used to choose service we want to load.
"""
self.__handlers = {}
self.__objectLoader = ObjectLoader()
self.__autoDiscovery = autoDiscovery
self.loader = ModuleLoader("Service", PathFinder.getServiceSection, RequestHandler, moduleSuffix="Handler")
def __loadLogClass(self, modulePath):
"""Load class thread-safe."""
# import ObjectLoader here to avoid a dependancy loop
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
objLoader = ObjectLoader()
# lock to avoid problem in ObjectLoader which is a singleton not
# thread-safe
self._lockObjectLoader.acquire()
try:
# load the Backend class
return objLoader.loadObject(modulePath)
finally:
self._lockObjectLoader.release()
return S_ERROR()
def getMQConnectorClass(mqType):
"""Function loads the specialized MQConnector class based on mqType.
It is assumed that MQConnector has a name in the format mqTypeMQConnector
e.g. if StompMQConnector.
Args:
mqType(str): prefix of specialized class name e.g. Stomp.
Returns:
S_OK or S_ERROR: with loaded specialized class of MQConnector.
"""
subClassName = mqType + "MQConnector"
result = ObjectLoader().loadObject("Resources.MessageQueue.%s" % subClassName)
if not result["OK"]:
gLogger.error("Failed to load object", "%s: %s" % (subClassName, result["Message"]))
return result
