def __init__(self, threads=8):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.fileset_watch = {}
self.last_poll_time = time.time()/60
self.workq = ThreadPool \
([self.pollExternal for _ in range(threads)])
def __init__(self, threads=8):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.fileset_watch = {}
self.last_poll_time = time.time() / 60
self.workq = ThreadPool \
([self.pollExternal for _ in range(threads)])
def __init__(self, slaveModule, component, \
threadPoolID= 'default', nrOfSlaves = 0):
"""
_init_
Initializes pool, and resets lost threads (e.g. lost during crash).
"""
#Queue.__init__(self, slaves)
Queue.__init__(self, [])
self.component = component
self.callQueue = 0
# we augment the threadpool id with component name in case
# we use separate talbes for the thread queues to prevent
# name clashes.
compName = self.component.config.Agent.componentName
self.threadPoolId = 'tp_threadpool_'+ \
compName+'_'+threadPoolID
# if set to false the peristent thread pool table is
# created per threadpool id.
self.oneQueue = True
# the size of the buffer to minimize single inserts in large tables.
self.bufferSize = 2
# the maximum number of slaves (threads) we allow.
self.nrOfSlaves = nrOfSlaves
self.slaveName = slaveModule.split(".")[-1]
self.slaveFactory = WMFactory("slaveFactory", \
slaveModule[0:slaveModule.rfind(slaveModule.split(".")[-1])-1])
myThread = threading.currentThread()
myThread.transaction.begin()
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
# check which tables we need to use.
self.poolTable = 'tp_threadpool'
self.poolTableBufferIn = 'tp_threadpool_buffer_in'
self.poolTableBufferOut = 'tp_threadpool_buffer_out'
if not self.oneQueue:
self.poolTable = self.threadPoolId
self.poolTableBufferIn = 'tp_threadpool_'+\
compName+'_'+\
threadPoolID+'_buffer_in'
self.poolTableBufferOut = 'tp_threadpool_'+\
compName+'_'+\
threadPoolID+'_buffer_out'
self.query.insertThreadPoolTables(self.poolTable)
self.query.insertThreadPoolTables(self.poolTableBufferIn)
self.query.insertThreadPoolTables(self.poolTableBufferOut)
# restore any threads that might have been lost during a crash
# de register thread in database so we do not need to restore it.
msg = "THREADPOOL: Resetting lost threads to queue status if any"
#logging.info(msg)
args = {'componentName' : compName, \
'thread_pool_id' : self.threadPoolId}
self.query.updateWorkStatus(args, self.poolTable)
self.query.updateWorkStatus(args, self.poolTableBufferIn)
self.query.updateWorkStatus(args, self.poolTableBufferOut)
# get the lenghts of the queue as this our starting point.
self.callQueue = self.countMessages()
# we do commit as initalization falls outside the while loop
# of the component.
myThread.transaction.commit()
class FeederManagerPoller(BaseWorkerThread):
"""
Regular managed fileset poller, instantiate feeder
by fileset
"""
def __init__(self, threads=8):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.fileset_watch = {}
self.last_poll_time = time.time()/60
self.workq = ThreadPool \
([self.pollExternal for _ in range(threads)])
def setup(self, parameters):
"""
Load DB objects required for queries
"""
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.FeederManager.Database" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
self.getAllManagedFilesets = daofactory(classname = "GetAllManagedFilesets")
self.getFeederId = daofactory(classname = "GetFeederId")
self.removeManagedFilesets = daofactory(classname = "RemoveManagedFilesets")
def databaseWork(self):
"""
completed, set the fileset to close (Not implemented yet)
"""
fileset_watch_temp = []
listFileset = {}
fileset_new = {}
myThread = threading.currentThread()
myThread.transaction.begin()
# Get All managed filesets
managedFilesets = self.getAllManagedFilesets.execute(\
conn = myThread.transaction.conn, transaction = True)
myThread.transaction.commit()
logging.debug("Found %s managed filesets" % len(managedFilesets))
for fileset in range(len(managedFilesets)):
logging.debug("Processing %s %s" % \
( managedFilesets[fileset]['id'] , managedFilesets[fileset]['name'] ) )
filesetToUpdate = Fileset(id=managedFilesets[fileset]['id'])
filesetToUpdate.load()
if managedFilesets[fileset]['name'] not in self.fileset_watch:
self.fileset_watch[filesetToUpdate.name] = filesetToUpdate
fileset_new[filesetToUpdate.name] = filesetToUpdate
listFileset[filesetToUpdate.name] = filesetToUpdate
# Update the list of the fileset to watch
for oldFileset in self.fileset_watch:
if oldFileset not in listFileset:
fileset_watch_temp.append(oldFileset)
# Remove from the list of the fileset to update the ones which are not
# in ManagedFilesets anymore
for oldTempFileset in fileset_watch_temp:
del self.fileset_watch[oldTempFileset]
logging.debug("NEW FILESETS %s" %fileset_new)
logging.debug("OLD FILESETS %s" %self.fileset_watch)
# WorkQueue work
for name, fileset in fileset_new.items():
logging.debug("Will poll %s : %s" % (name, fileset.id))
self.workq.enqueue(name, fileset)
for key, filesets in self.workq.__iter__():
fileset = self.fileset_watch[key]
logging.debug \
("the poll key %s result %s is ready !" % (key, str(fileset.id)))
myThread.transaction.begin()
feederId = self.getFeederId.execute( \
feederType = (fileset.name).split(":")[1], \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
logging.debug("the Feeder %s has processed %s and is \
removing it if closed" % (feederId, fileset.name) )
# Finally delete fileset
# If the fileset is closed remove it
fileset.load()
if fileset.open == False:
myThread.transaction.begin()
self.removeManagedFilesets.execute( \
filesetId = fileset.id, feederType = feederId, \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
# Handles old filesets. We update old filesets every 10 mn
# We need to make old filesets update cycle configurable
if ((time.time()/60) - self.last_poll_time) > 10 :
# WorkQueue handles old filesets
for name, fileset in self.fileset_watch.items():
logging.debug("Will poll %s : %s" % (name, fileset.id))
self.workq.enqueue(name, fileset)
for key, filesets in self.workq.__iter__():
fileset = self.fileset_watch[key]
logging.debug \
("the poll key %s result %s is ready !" % (key, str(fileset.id)))
myThread.transaction.begin()
feederId = self.getFeederId.execute(\
feederType = (fileset.name).split(":")[1], \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
logging.debug("the Feeder %s has processed %s and is \
removing it if closed" % (feederId, fileset.name) )
# Finally delete fileset
# If the fileset is closed remove it
fileset.load()
if fileset.open == False:
myThread.transaction.begin()
self.removeManagedFilesets.execute(\
filesetId = fileset.id, feederType = feederId, \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
# Update the last update time of old filesets
self.last_poll_time = time.time()/60
# Switch to static method if needed
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject( classname = "Feeder", getFromCache = False )
feeder(fileset)
except Exception as e :
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info( traceback.format_exc() )
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def algorithm(self, parameters):
"""
Queries DB for all watched filesets, if information about filesets are
completed delete it from watched filesets (set the fileset to close)
"""
logging.debug("Running feeder / fileset completion algorithm")
myThread = threading.currentThread()
try:
myThread.transaction.begin()
self.databaseWork()
myThread.transaction.commit()
except:
myThread.transaction.rollback()
raise
def __init__(self, slaveModule, component, \
threadPoolID= 'default', nrOfSlaves = 0):
"""
_init_
Initializes pool, and resets lost threads (e.g. lost during crash).
"""
#Queue.__init__(self, slaves)
Queue.__init__(self, [])
self.component = component
self.callQueue = 0
# we augment the threadpool id with component name in case
# we use separate talbes for the thread queues to prevent
# name clashes.
compName = self.component.config.Agent.componentName
self.threadPoolId = 'tp_threadpool_'+ \
compName+'_'+threadPoolID
# if set to false the peristent thread pool table is
# created per threadpool id.
self.oneQueue = True
# the size of the buffer to minimize single inserts in large tables.
self.bufferSize = 2
# the maximum number of slaves (threads) we allow.
self.nrOfSlaves = nrOfSlaves
self.slaveName = slaveModule.split(".")[-1]
self.slaveFactory = WMFactory("slaveFactory", \
slaveModule[0:slaveModule.rfind(slaveModule.split(".")[-1])-1])
myThread = threading.currentThread()
myThread.transaction.begin()
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
# check which tables we need to use.
self.poolTable = 'tp_threadpool'
self.poolTableBufferIn = 'tp_threadpool_buffer_in'
self.poolTableBufferOut = 'tp_threadpool_buffer_out'
if not self.oneQueue:
self.poolTable = self.threadPoolId
self.poolTableBufferIn = 'tp_threadpool_'+\
compName+'_'+\
threadPoolID+'_buffer_in'
self.poolTableBufferOut = 'tp_threadpool_'+\
compName+'_'+\
threadPoolID+'_buffer_out'
self.query.insertThreadPoolTables(self.poolTable)
self.query.insertThreadPoolTables(self.poolTableBufferIn)
self.query.insertThreadPoolTables(self.poolTableBufferOut)
# restore any threads that might have been lost during a crash
# de register thread in database so we do not need to restore it.
msg = "THREADPOOL: Resetting lost threads to queue status if any"
#logging.info(msg)
args = {'componentName' : compName, \
'thread_pool_id' : self.threadPoolId}
self.query.updateWorkStatus(args, self.poolTable)
self.query.updateWorkStatus(args, self.poolTableBufferIn)
self.query.updateWorkStatus(args, self.poolTableBufferOut)
# get the lenghts of the queue as this our starting point.
self.callQueue = self.countMessages()
# we do commit as initalization falls outside the while loop
# of the component.
myThread.transaction.commit()
class FeederManagerPoller(BaseWorkerThread):
"""
Regular managed fileset poller, instantiate feeder
by fileset
"""
def __init__(self, threads=8):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.fileset_watch = {}
self.last_poll_time = time.time() / 60
self.workq = ThreadPool \
([self.pollExternal for _ in range(threads)])
def setup(self, parameters):
"""
Load DB objects required for queries
"""
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMComponent.FeederManager.Database" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
self.getAllManagedFilesets = daofactory(
classname="GetAllManagedFilesets")
self.getFeederId = daofactory(classname="GetFeederId")
self.removeManagedFilesets = daofactory(
classname="RemoveManagedFilesets")
def databaseWork(self):
"""
completed, set the fileset to close (Not implemented yet)
"""
fileset_watch_temp = []
listFileset = {}
fileset_new = {}
myThread = threading.currentThread()
myThread.transaction.begin()
# Get All managed filesets
managedFilesets = self.getAllManagedFilesets.execute(\
conn = myThread.transaction.conn, transaction = True)
myThread.transaction.commit()
logging.debug("Found %s managed filesets" % len(managedFilesets))
for fileset in range(len(managedFilesets)):
logging.debug("Processing %s %s" % \
( managedFilesets[fileset]['id'] , managedFilesets[fileset]['name'] ) )
filesetToUpdate = Fileset(id=managedFilesets[fileset]['id'])
filesetToUpdate.load()
if managedFilesets[fileset]['name'] not in self.fileset_watch:
self.fileset_watch[filesetToUpdate.name] = filesetToUpdate
fileset_new[filesetToUpdate.name] = filesetToUpdate
listFileset[filesetToUpdate.name] = filesetToUpdate
# Update the list of the fileset to watch
for oldFileset in self.fileset_watch:
if oldFileset not in listFileset:
fileset_watch_temp.append(oldFileset)
# Remove from the list of the fileset to update the ones which are not
# in ManagedFilesets anymore
for oldTempFileset in fileset_watch_temp:
del self.fileset_watch[oldTempFileset]
logging.debug("NEW FILESETS %s" % fileset_new)
logging.debug("OLD FILESETS %s" % self.fileset_watch)
# WorkQueue work
for name, fileset in fileset_new.items():
logging.debug("Will poll %s : %s" % (name, fileset.id))
self.workq.enqueue(name, fileset)
for key, filesets in self.workq.__iter__():
fileset = self.fileset_watch[key]
logging.debug \
("the poll key %s result %s is ready !" % (key, str(fileset.id)))
myThread.transaction.begin()
feederId = self.getFeederId.execute( \
feederType = (fileset.name).split(":")[1], \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
logging.debug("the Feeder %s has processed %s and is \
removing it if closed" % (feederId, fileset.name))
# Finally delete fileset
# If the fileset is closed remove it
fileset.load()
if fileset.open == False:
myThread.transaction.begin()
self.removeManagedFilesets.execute( \
filesetId = fileset.id, feederType = feederId, \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
# Handles old filesets. We update old filesets every 10 mn
# We need to make old filesets update cycle configurable
if ((time.time() / 60) - self.last_poll_time) > 10:
# WorkQueue handles old filesets
for name, fileset in self.fileset_watch.items():
logging.debug("Will poll %s : %s" % (name, fileset.id))
self.workq.enqueue(name, fileset)
for key, filesets in self.workq.__iter__():
fileset = self.fileset_watch[key]
logging.debug \
("the poll key %s result %s is ready !" % (key, str(fileset.id)))
myThread.transaction.begin()
feederId = self.getFeederId.execute(\
feederType = (fileset.name).split(":")[1], \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
logging.debug("the Feeder %s has processed %s and is \
removing it if closed" % (feederId, fileset.name))
# Finally delete fileset
# If the fileset is closed remove it
fileset.load()
if fileset.open == False:
myThread.transaction.begin()
self.removeManagedFilesets.execute(\
filesetId = fileset.id, feederType = feederId, \
conn = myThread.transaction.conn, transaction = True )
myThread.transaction.commit()
# Update the last update time of old filesets
self.last_poll_time = time.time() / 60
# Switch to static method if needed
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject(classname="Feeder", getFromCache=False)
feeder(fileset)
except Exception as e:
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info(traceback.format_exc())
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def algorithm(self, parameters):
"""
Queries DB for all watched filesets, if information about filesets are
completed delete it from watched filesets (set the fileset to close)
"""
logging.debug("Running feeder / fileset completion algorithm")
myThread = threading.currentThread()
try:
myThread.transaction.begin()
self.databaseWork()
myThread.transaction.commit()
except:
myThread.transaction.rollback()
raise
