def export_requestJob(self, resourceDescription): """ Serve a job to the request of an agent which is the highest priority one matching the agent's site capacity """ resourceDescription['Setup'] = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() try: opsHelper = Operations(group=credDict['group']) matcher = Matcher(pilotAgentsDB=pilotAgentsDB, jobDB=gJobDB, tqDB=gTaskQueueDB, jlDB=jlDB, opsHelper=opsHelper) result = matcher.selectJob(resourceDescription, credDict) except RuntimeError as rte: self.log.error("Error requesting job: ", rte) return S_ERROR("Error requesting job") # result can be empty, meaning that no job matched if result: gMonitor.addMark("matchesDone") gMonitor.addMark("matchesOK") return S_OK(result) # FIXME: This is correctly interpreted by the JobAgent, but DErrno should be used instead return S_ERROR("No match found")
def export_requestJob(self, resourceDescription): """Serve a job to the request of an agent which is the highest priority one matching the agent's site capacity """ resourceDescription["Setup"] = self.serviceInfoDict["clientSetup"] credDict = self.getRemoteCredentials() pilotRef = resourceDescription.get("PilotReference", "Unknown") try: opsHelper = Operations(group=credDict["group"]) matcher = Matcher( pilotAgentsDB=self.pilotAgentsDB, jobDB=self.jobDB, tqDB=self.taskQueueDB, jlDB=self.jobLoggingDB, opsHelper=opsHelper, pilotRef=pilotRef, ) result = matcher.selectJob(resourceDescription, credDict) except RuntimeError as rte: self.log.error("Error requesting job for pilot", "[%s] %s" % (pilotRef, rte)) return S_ERROR("Error requesting job") except PilotVersionError as pve: self.log.warn("Pilot version error for pilot", "[%s] %s" % (pilotRef, pve)) return S_ERROR(DErrno.EWMSPLTVER, callStack=[]) # result can be empty, meaning that no job matched if result: return S_OK(result) return S_ERROR(DErrno.EWMSNOMATCH, callStack=[])
def export_getMatchingTaskQueues(cls, resourceDict): """Return all task queues that match the resourceDict""" if "Site" in resourceDict and isinstance(resourceDict["Site"], six.string_types): gridCE = resourceDict.get("GridCE") negativeCond = cls.limiter.getNegativeCondForSite(resourceDict["Site"], gridCE) else: negativeCond = cls.limiter.getNegativeCond() matcher = Matcher(pilotAgentsDB=cls.pilotAgentsDB, jobDB=cls.jobDB, tqDB=cls.taskQueueDB, jlDB=cls.jobLoggingDB) resourceDescriptionDict = matcher._processResourceDescription(resourceDict) return cls.taskQueueDB.getMatchingTaskQueues(resourceDescriptionDict, negativeCond=negativeCond)
def export_getMatchingTaskQueues(self, resourceDict): """ Return all task queues """ if 'Site' in resourceDict and isinstance(resourceDict['Site'], basestring): negativeCond = self.limiter.getNegativeCondForSite(resourceDict['Site']) else: negativeCond = self.limiter.getNegativeCond() matcher = Matcher(pilotAgentsDB=pilotAgentsDB, jobDB=gJobDB, tqDB=gTaskQueueDB, jlDB=jlDB) resourceDescriptionDict = matcher._processResourceDescription(resourceDict) return gTaskQueueDB.retrieveTaskQueuesThatMatch(resourceDescriptionDict, negativeCond=negativeCond)
def export_getMatchingTaskQueues(self, resourceDict): """ Return all task queues that match the resourceDict """ if 'Site' in resourceDict and isinstance(resourceDict['Site'], six.string_types): negativeCond = self.limiter.getNegativeCondForSite(resourceDict['Site']) else: negativeCond = self.limiter.getNegativeCond() matcher = Matcher(pilotAgentsDB=pilotAgentsDB, jobDB=gJobDB, tqDB=gTaskQueueDB, jlDB=jlDB) resourceDescriptionDict = matcher._processResourceDescription(resourceDict) return gTaskQueueDB.getMatchingTaskQueues(resourceDescriptionDict, negativeCond=negativeCond)
def export_requestJob(self, resourceDescription): """ Serve a job to the request of an agent which is the highest priority one matching the agent's site capacity """ resourceDescription['Setup'] = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() try: opsHelper = Operations(group=credDict['group']) matcher = Matcher(pilotAgentsDB=pilotAgentsDB, jobDB=gJobDB, tqDB=gTaskQueueDB, jlDB=jlDB, opsHelper=opsHelper) result = matcher.selectJob(resourceDescription, credDict) except RuntimeError, rte: self.log.error("Error requesting job: ", rte) return S_ERROR("Error requesting job")
def export_requestJob( self, resourceDescription ): """ Serve a job to the request of an agent which is the highest priority one matching the agent's site capacity """ resourceDescription['Setup'] = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() try: opsHelper = Operations( group = credDict['group'] ) matcher = Matcher( pilotAgentsDB = pilotAgentsDB, jobDB = gJobDB, tqDB = gTaskQueueDB, jlDB = jlDB, opsHelper = opsHelper ) result = matcher.selectJob( resourceDescription, credDict ) except RuntimeError, rte: self.log.error( "Error requesting job: ", rte ) return S_ERROR( "Error requesting job" )
def setUp(self): from DIRAC import gLogger gLogger.setLevel('DEBUG') self.mockDM = MagicMock() self.mockDM.return_value = dm_mock mockObjectSE = MagicMock() mockObjectSE.getFileMetadata.return_value = S_OK({ 'Successful': { '/a/lfn/1.txt': { 'Cached': 0 }, '/a/lfn/2.txt': { 'Cached': 1 } }, 'Failed': {} }) mockObjectSE.getFile.return_value = S_OK({ 'Successful': { '/a/lfn/1.txt': {} }, 'Failed': {} }) mockObjectSE.getStatus.return_value = S_OK({ 'Read': True, 'DiskSE': True }) self.mockSE = MagicMock() self.mockSE.return_value = mockObjectSE self.dli = DownloadInputData({ 'InputData': [], 'Configuration': 'boh', 'FileCatalog': S_OK({'Successful': []}) }) self.pilotAgentsDBMock = MagicMock() self.jobDBMock = MagicMock() self.tqDBMock = MagicMock() self.jlDBMock = MagicMock() self.opsHelperMock = MagicMock() self.matcher = Matcher(pilotAgentsDB=self.pilotAgentsDBMock, jobDB=self.jobDBMock, tqDB=self.tqDBMock, jlDB=self.jlDBMock, opsHelper=self.opsHelperMock)
def setUp(self): from DIRAC import gLogger gLogger.setLevel('DEBUG') self.pilotAgentsDBMock = MagicMock() self.jobDBMock = MagicMock() self.tqDBMock = MagicMock() self.jlDBMock = MagicMock() self.opsHelperMock = MagicMock() self.matcher = Matcher(pilotAgentsDB=self.pilotAgentsDBMock, jobDB=self.jobDBMock, tqDB=self.tqDBMock, jlDB=self.jlDBMock, opsHelper=self.opsHelperMock)
class MatcherHandler( RequestHandler ): def initialize( self ): self.matcher = Matcher( pilotAgentsDB = pilotAgentsDB, jobDB = gJobDB, tqDB = gTaskQueueDB, jlDB = jlDB, ) self.limiter = Limiter( jobDB = gJobDB ) ############################################################################## types_requestJob = [ [StringType, DictType] ] def export_requestJob( self, resourceDescription ): """ Serve a job to the request of an agent which is the highest priority one matching the agent's site capacity """ resourceDescription['Setup'] = self.serviceInfoDict['clientSetup'] credDict = self.getRemoteCredentials() try: result = self.matcher.selectJob( resourceDescription, credDict ) except RuntimeError, rte: self.log.error( "Error requesting job: ", rte ) return S_ERROR( "Error requesting job" ) gMonitor.addMark( "matchesDone" ) gMonitor.addMark( "matchesOK" ) return S_OK( result )
gLogger.setLevel("DEBUG") # sut from DIRAC.WorkloadManagementSystem.Client.Matcher import Matcher from DIRAC.WorkloadManagementSystem.Client.SandboxStoreClient import SandboxStoreClient pilotAgentsDBMock = MagicMock() jobDBMock = MagicMock() tqDBMock = MagicMock() jlDBMock = MagicMock() opsHelperMock = MagicMock() matcher = Matcher( pilotAgentsDB=pilotAgentsDBMock, jobDB=jobDBMock, tqDB=tqDBMock, jlDB=jlDBMock, opsHelper=opsHelperMock, ) @pytest.fixture def setUp(): yield setUp try: os.remove("1.txt") os.remove("InputData_*") except OSError: pass
def initialize( self ): self.matcher = Matcher( pilotAgentsDB = pilotAgentsDB, jobDB = gJobDB, tqDB = gTaskQueueDB, jlDB = jlDB, ) self.limiter = Limiter( jobDB = gJobDB )