class MakeJob(BaseHandler):
"""
Default handler for Make Job Events.
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool (this is optional
# and depends on the developer deciding how he/she
# wants to implement certain logic.
self.nThreads = 5
self.threadpool = ThreadPool(\
"WMCore.WMSpec.Makers.Handlers.MakeJobSlave", \
self.component, 'MakeJob', \
self.nThreads)
myThread = threading.currentThread()
myThread.msgService.purgeMessages()
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# as we defined a threadpool we can enqueue our item
# and move to the next.
self.threadpool.enqueue(event, payload)
print "Thread with payload " + str(payload) + " is enqueued"
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool to handle messages
self.threadpool = ThreadPool(\
"WMComponent.FeederManager.Handler.DefaultAddDatasetWatchSlave", \
self.component, 'AddDatasetWatch', \
self.component.config.FeederManager.maxThreads)
class MakeJob(BaseHandler):
"""
Default handler for Make Job Events.
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool (this is optional
# and depends on the developer deciding how he/she
# wants to implement certain logic.
self.nThreads = 5
self.threadpool = ThreadPool(\
"WMCore.WMSpec.Makers.Handlers.MakeJobSlave", \
self.component, 'MakeJob', \
self.nThreads)
myThread = threading.currentThread()
myThread.msgService.purgeMessages()
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# as we defined a threadpool we can enqueue our item
# and move to the next.
self.threadpool.enqueue(event, payload)
print("Thread with payload " + str(payload) + " is enqueued")
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManageSlave", \
self.component, 'AddWorkflowToManage', \
self.component.config.WorkflowManager.maxThreads)
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationListSlave", \
self.component, 'AddToWorkflowManagementLocationList', \
self.component.config.WorkflowManager.maxThreads)
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagementSlave", \
self.component, 'RemoveWorkflowFromManagement', \
self.component.config.WorkflowManager.maxThreads)
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool (this is optional
# and depends on the developer deciding how he/she
# wants to implement certain logic.
self.nThreads = 5
self.threadpool = ThreadPool(\
"WMCore.WMSpec.Makers.Handlers.MakeJobSlave", \
self.component, 'MakeJob', \
self.nThreads)
myThread = threading.currentThread()
myThread.msgService.purgeMessages()
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool to handle messages
self.threadpool = ThreadPool(\
"WMComponent.FeederManager.Handler.DefaultAddDatasetWatchSlave", \
self.component, 'AddDatasetWatch', \
self.component.config.FeederManager.maxThreads)
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManageSlave", \
self.component, 'AddWorkflowToManage', \
self.component.config.WorkflowManager.maxThreads)
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool (this is optional
# and depends on the developer deciding how he/she
# wants to implement certain logic.
self.threadpool = ThreadPool(\
"TQComp.Handler.TaskHandlerSlave", \
self.component, 'default', \
self.component.config.TQComp.maxThreads)
class RemoveWorkflowFromManagement(BaseHandler):
"""
Default handler for removal of workflow / fileset --> workflow mapping
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagementSlave", \
self.component, 'RemoveWorkflowFromManagement', \
self.component.config.WorkflowManager.maxThreads)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# Add item to the thread pool and return
self.threadpool.enqueue(event, {'event' : event, 'payload' :payload['payload']})
class RemoveFromWorkflowManagementLocationList(BaseHandler):
"""
Default handler for removal of location information for created
subscriptions
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationListSlave", \
self.component, 'RemoveFromWorkflowManagementLocationList', \
self.component.config.WorkflowManager.maxThreads)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# Add item to the thread pool and return
self.threadpool.enqueue(event, {'event': event, 'payload': payload})
class AddToWorkflowManagementLocationList(BaseHandler):
"""
Default handler for addition of location information for created
subscriptions
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationListSlave", \
self.component, 'AddToWorkflowManagementLocationList', \
self.component.config.WorkflowManager.maxThreads)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# Add item to the thread pool and return
self.threadpool.enqueue(event, {'event' : event, 'payload' :payload['payload']})
class DefaultAddDatasetWatch(BaseHandler):
"""
Default handler for dataset watch
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool to handle messages
self.threadpool = ThreadPool(\
"WMComponent.FeederManager.Handler.DefaultAddDatasetWatchSlave", \
self.component, 'AddDatasetWatch', \
self.component.config.FeederManager.maxThreads)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# as we defined a threadpool we can enqueue our item
# and move to the next.
self.threadpool.enqueue(event, {'event' : event, 'payload' :payload['payload']})
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool (this is optional
# and depends on the developer deciding how he/she
# wants to implement certain logic.
self.nThreads = 5
self.threadpool = ThreadPool(\
"WMCore.WMSpec.Makers.Handlers.MakeJobSlave", \
self.component, 'MakeJob', \
self.nThreads)
myThread = threading.currentThread()
myThread.msgService.purgeMessages()
class TaskHandler(BaseHandler):
"""
Base handler for new tasks.
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool (this is optional
# and depends on the developer deciding how he/she
# wants to implement certain logic.
self.threadpool = ThreadPool(\
"TQComp.Handler.TaskHandlerSlave", \
self.component, 'default', \
self.component.config.TQComp.maxThreads)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# as we defined a threadpool we can enqueue our item
# and move to the next.
self.threadpool.enqueue(event, {'event' : event, 'payload' :payload})
class AddWorkflowToManage(BaseHandler):
"""
Default handler for addition of workflow / fileset --> workflow mapping
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# Define a slave threadpool
self.threadpool = ThreadPool(
"WMComponent.WorkflowManager.Handler.AddWorkflowToManageSlave",
self.component,
"AddWorkflowToManage",
self.component.config.WorkflowManager.maxThreads,
)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# Add item to the thread pool and return
self.threadpool.enqueue(event, {"event": event, "payload": payload["payload"]})
class DefaultAddDatasetWatch(BaseHandler):
"""
Default handler for dataset watch
"""
def __init__(self, component):
BaseHandler.__init__(self, component)
# define a slave threadpool to handle messages
self.threadpool = ThreadPool(\
"WMComponent.FeederManager.Handler.DefaultAddDatasetWatchSlave", \
self.component, 'AddDatasetWatch', \
self.component.config.FeederManager.maxThreads)
# this we overload from the base handler
def __call__(self, event, payload):
"""
Handles the event with payload, by sending it to the threadpool.
"""
# as we defined a threadpool we can enqueue our item
# and move to the next.
self.threadpool.enqueue(event, {
'event': event,
'payload': payload['payload']
})
def testA(self):
"""
__testSubscribe__
Test subscription of a component.
"""
raise nose.SkipTest
myThread = threading.currentThread()
# create a 'fake' component that contains a arg dictionary.
component = Dummy()
# load default parameters.
config = self.testInit.getConfiguration()
# normally assigned by the harness of the test component.
config.Agent.componentName = "TestComponent"
component.config = config
threadPools = []
for i in xrange(0, ThreadPoolTest._nrOfPools):
threadPool = ThreadPool("WMCore.ThreadPool.ThreadSlave", \
component, 'MyPool_'+str(i), ThreadPoolTest._nrOfThreads)
threadPools.append(threadPool)
# this is how you would use the threadpool. The threadpool retrieves
# events/payloads from the message service. If a thread is available
# it is dispatched, otherwise it is stored in the trheadpool.
# make the number of tasks bigger than number of threads to tesT
# the persistent queue.
for i in xrange(0, ThreadPoolTest._nrOfThreads * 10):
event = 'eventNr_' + str(i)
payload = 'payloadNr_' + str(i)
# normally you would have different events per threadpool and
# even different objects per pool. the payload part will be
# pickled into the database enabling flexibility in passing
# information.
for j in xrange(0, ThreadPoolTest._nrOfPools):
threadPools[j].enqueue(event, \
{'event' : event, 'payload' : payload})
# this commit you want to be in the agent harness, so the message is
# actual removed from the msgService. we can do this as the threadpool
# acts as a dispatcher and is a shortlived action: dispatch to thread
# or queue and tell agent harness it is finished.
finished = False
timeout = 60 # secs
currenttime = 0
while not finished:
print('waiting for threads to finishs. Work left:')
for j in xrange(0, ThreadPoolTest._nrOfPools):
print('pool_' + str(j) + ':' + str(threadPools[j].callQueue))
time.sleep(1)
finished = True
currenttime += 1
if (timeout == currenttime):
raise RuntimeError
for j in xrange(0, ThreadPoolTest._nrOfPools):
if (len(threadPools[j].resultsQueue) <
ThreadPoolTest._nrOfThreads * 10):
finished = False
break
# check if the tables are really empty and all messages
# have been processed.
for j in xrange(0, ThreadPoolTest._nrOfPools):
assert len(threadPools[j].resultsQueue) == \
ThreadPoolTest._nrOfThreads*10
myThread.transaction.begin()
for j in xrange(0, ThreadPoolTest._nrOfPools):
self.assertEqual(threadPools[j].countMessages(), 0)
myThread.transaction.commit()
