def export_requestJob(self, resourceDescription):
""" Serve a job to the request of an agent which is the highest priority
one matching the agent's site capacity
"""
resourceDescription['Setup'] = self.serviceInfoDict['clientSetup']
credDict = self.getRemoteCredentials()
try:
opsHelper = Operations(group=credDict['group'])
matcher = Matcher(pilotAgentsDB=pilotAgentsDB,
jobDB=gJobDB,
tqDB=gTaskQueueDB,
jlDB=jlDB,
opsHelper=opsHelper)
result = matcher.selectJob(resourceDescription, credDict)
except RuntimeError as rte:
self.log.error("Error requesting job: ", rte)
return S_ERROR("Error requesting job")
# result can be empty, meaning that no job matched
if result:
gMonitor.addMark("matchesDone")
gMonitor.addMark("matchesOK")
return S_OK(result)
# FIXME: This is correctly interpreted by the JobAgent, but DErrno should be used instead
return S_ERROR("No match found")
def export_requestJob(self, resourceDescription):
"""Serve a job to the request of an agent which is the highest priority
one matching the agent's site capacity
"""
resourceDescription["Setup"] = self.serviceInfoDict["clientSetup"]
credDict = self.getRemoteCredentials()
pilotRef = resourceDescription.get("PilotReference", "Unknown")
try:
opsHelper = Operations(group=credDict["group"])
matcher = Matcher(
pilotAgentsDB=self.pilotAgentsDB,
jobDB=self.jobDB,
tqDB=self.taskQueueDB,
jlDB=self.jobLoggingDB,
opsHelper=opsHelper,
pilotRef=pilotRef,
)
result = matcher.selectJob(resourceDescription, credDict)
except RuntimeError as rte:
self.log.error("Error requesting job for pilot",
"[%s] %s" % (pilotRef, rte))
return S_ERROR("Error requesting job")
except PilotVersionError as pve:
self.log.warn("Pilot version error for pilot",
"[%s] %s" % (pilotRef, pve))
return S_ERROR(DErrno.EWMSPLTVER, callStack=[])
# result can be empty, meaning that no job matched
if result:
return S_OK(result)
return S_ERROR(DErrno.EWMSNOMATCH, callStack=[])
def export_requestJob(self, resourceDescription):
""" Serve a job to the request of an agent which is the highest priority
one matching the agent's site capacity
"""
resourceDescription['Setup'] = self.serviceInfoDict['clientSetup']
credDict = self.getRemoteCredentials()
try:
opsHelper = Operations(group=credDict['group'])
matcher = Matcher(pilotAgentsDB=pilotAgentsDB,
jobDB=gJobDB,
tqDB=gTaskQueueDB,
jlDB=jlDB,
opsHelper=opsHelper)
result = matcher.selectJob(resourceDescription, credDict)
except RuntimeError as rte:
self.log.error("Error requesting job: ", rte)
return S_ERROR("Error requesting job")
# result can be empty, meaning that no job matched
if result:
gMonitor.addMark("matchesDone")
gMonitor.addMark("matchesOK")
return S_OK(result)
# FIXME: This is correctly interpreted by the JobAgent, but DErrno should be used instead
return S_ERROR("No match found")
def export_getMatchingTaskQueues(cls, resourceDict):
"""Return all task queues that match the resourceDict"""
if "Site" in resourceDict and isinstance(resourceDict["Site"], six.string_types):
gridCE = resourceDict.get("GridCE")
negativeCond = cls.limiter.getNegativeCondForSite(resourceDict["Site"], gridCE)
else:
negativeCond = cls.limiter.getNegativeCond()
matcher = Matcher(pilotAgentsDB=cls.pilotAgentsDB, jobDB=cls.jobDB, tqDB=cls.taskQueueDB, jlDB=cls.jobLoggingDB)
resourceDescriptionDict = matcher._processResourceDescription(resourceDict)
return cls.taskQueueDB.getMatchingTaskQueues(resourceDescriptionDict, negativeCond=negativeCond)
def export_getMatchingTaskQueues(self, resourceDict):
""" Return all task queues
"""
if 'Site' in resourceDict and isinstance(resourceDict['Site'], basestring):
negativeCond = self.limiter.getNegativeCondForSite(resourceDict['Site'])
else:
negativeCond = self.limiter.getNegativeCond()
matcher = Matcher(pilotAgentsDB=pilotAgentsDB,
jobDB=gJobDB,
tqDB=gTaskQueueDB,
jlDB=jlDB)
resourceDescriptionDict = matcher._processResourceDescription(resourceDict)
return gTaskQueueDB.retrieveTaskQueuesThatMatch(resourceDescriptionDict, negativeCond=negativeCond)
def export_getMatchingTaskQueues(self, resourceDict):
""" Return all task queues that match the resourceDict
"""
if 'Site' in resourceDict and isinstance(resourceDict['Site'], six.string_types):
negativeCond = self.limiter.getNegativeCondForSite(resourceDict['Site'])
else:
negativeCond = self.limiter.getNegativeCond()
matcher = Matcher(pilotAgentsDB=pilotAgentsDB,
jobDB=gJobDB,
tqDB=gTaskQueueDB,
jlDB=jlDB)
resourceDescriptionDict = matcher._processResourceDescription(resourceDict)
return gTaskQueueDB.getMatchingTaskQueues(resourceDescriptionDict,
negativeCond=negativeCond)
def export_requestJob(self, resourceDescription):
""" Serve a job to the request of an agent which is the highest priority
one matching the agent's site capacity
"""
resourceDescription['Setup'] = self.serviceInfoDict['clientSetup']
credDict = self.getRemoteCredentials()
try:
opsHelper = Operations(group=credDict['group'])
matcher = Matcher(pilotAgentsDB=pilotAgentsDB,
jobDB=gJobDB,
tqDB=gTaskQueueDB,
jlDB=jlDB,
opsHelper=opsHelper)
result = matcher.selectJob(resourceDescription, credDict)
except RuntimeError, rte:
self.log.error("Error requesting job: ", rte)
return S_ERROR("Error requesting job")
def export_requestJob( self, resourceDescription ):
""" Serve a job to the request of an agent which is the highest priority
one matching the agent's site capacity
"""
resourceDescription['Setup'] = self.serviceInfoDict['clientSetup']
credDict = self.getRemoteCredentials()
try:
opsHelper = Operations( group = credDict['group'] )
matcher = Matcher( pilotAgentsDB = pilotAgentsDB,
jobDB = gJobDB,
tqDB = gTaskQueueDB,
jlDB = jlDB,
opsHelper = opsHelper )
result = matcher.selectJob( resourceDescription, credDict )
except RuntimeError, rte:
self.log.error( "Error requesting job: ", rte )
return S_ERROR( "Error requesting job" )
def setUp(self):
from DIRAC import gLogger
gLogger.setLevel('DEBUG')
self.mockDM = MagicMock()
self.mockDM.return_value = dm_mock
mockObjectSE = MagicMock()
mockObjectSE.getFileMetadata.return_value = S_OK({
'Successful': {
'/a/lfn/1.txt': {
'Cached': 0
},
'/a/lfn/2.txt': {
'Cached': 1
}
},
'Failed': {}
})
mockObjectSE.getFile.return_value = S_OK({
'Successful': {
'/a/lfn/1.txt': {}
},
'Failed': {}
})
mockObjectSE.getStatus.return_value = S_OK({
'Read': True,
'DiskSE': True
})
self.mockSE = MagicMock()
self.mockSE.return_value = mockObjectSE
self.dli = DownloadInputData({
'InputData': [],
'Configuration': 'boh',
'FileCatalog': S_OK({'Successful': []})
})
self.pilotAgentsDBMock = MagicMock()
self.jobDBMock = MagicMock()
self.tqDBMock = MagicMock()
self.jlDBMock = MagicMock()
self.opsHelperMock = MagicMock()
self.matcher = Matcher(pilotAgentsDB=self.pilotAgentsDBMock,
jobDB=self.jobDBMock,
tqDB=self.tqDBMock,
jlDB=self.jlDBMock,
opsHelper=self.opsHelperMock)
def setUp(self):
from DIRAC import gLogger
gLogger.setLevel('DEBUG')
self.pilotAgentsDBMock = MagicMock()
self.jobDBMock = MagicMock()
self.tqDBMock = MagicMock()
self.jlDBMock = MagicMock()
self.opsHelperMock = MagicMock()
self.matcher = Matcher(pilotAgentsDB=self.pilotAgentsDBMock,
jobDB=self.jobDBMock,
tqDB=self.tqDBMock,
jlDB=self.jlDBMock,
opsHelper=self.opsHelperMock)
class MatcherHandler( RequestHandler ):
def initialize( self ):
self.matcher = Matcher( pilotAgentsDB = pilotAgentsDB, jobDB = gJobDB, tqDB = gTaskQueueDB, jlDB = jlDB, )
self.limiter = Limiter( jobDB = gJobDB )
##############################################################################
types_requestJob = [ [StringType, DictType] ]
def export_requestJob( self, resourceDescription ):
""" Serve a job to the request of an agent which is the highest priority
one matching the agent's site capacity
"""
resourceDescription['Setup'] = self.serviceInfoDict['clientSetup']
credDict = self.getRemoteCredentials()
try:
result = self.matcher.selectJob( resourceDescription, credDict )
except RuntimeError, rte:
self.log.error( "Error requesting job: ", rte )
return S_ERROR( "Error requesting job" )
gMonitor.addMark( "matchesDone" )
gMonitor.addMark( "matchesOK" )
return S_OK( result )
gLogger.setLevel("DEBUG")
# sut
from DIRAC.WorkloadManagementSystem.Client.Matcher import Matcher
from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient
pilotAgentsDBMock = MagicMock()
jobDBMock = MagicMock()
tqDBMock = MagicMock()
jlDBMock = MagicMock()
opsHelperMock = MagicMock()
matcher = Matcher(
pilotAgentsDB=pilotAgentsDBMock,
jobDB=jobDBMock,
tqDB=tqDBMock,
jlDB=jlDBMock,
opsHelper=opsHelperMock,
)
@pytest.fixture
def setUp():
yield setUp
try:
os.remove("1.txt")
os.remove("InputData_*")
except OSError:
pass
def initialize( self ): self.matcher = Matcher( pilotAgentsDB = pilotAgentsDB, jobDB = gJobDB, tqDB = gTaskQueueDB, jlDB = jlDB, ) self.limiter = Limiter( jobDB = gJobDB )
