def preInitialization(self):
"""
Add required worker modules to work threads
"""
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['AddWorkflowToManage'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManage", self)
self.messages['RemoveWorkflowFromManagement'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagement", self)
self.messages['AddToWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationList", self)
self.messages['RemoveFromWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationList", self)
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkflowManager.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(WorkflowManagerPoller(), \
pollInterval)
def preInitialization(self):
"""
Add required worker modules to work threads
"""
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['AddWorkflowToManage'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManage", self)
self.messages['RemoveWorkflowFromManagement'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagement", self)
self.messages['AddToWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationList", self)
self.messages['RemoveFromWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationList", self)
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkflowManager.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(WorkflowManagerPoller(), \
pollInterval)
class Plotter(RESTModel):
'''
A class that generates a plot object to send to the formatter, possibly
downloading the data from a remote resource along the way
'''
def __init__(self, config):
RESTModel.__init__(self, config)
self.methods['POST'] = {'plot': {'version':1,
'call':self.plot,
'args': ['type', 'data', 'url'],
'expires': 300}}
self.methods['GET'] = {'plot': {'version':1,
'call':self.plot,
'args': ['type', 'data', 'url'],
'expires': 300},
'doc': {'version':1,
'call':self.doc,
'args': ['type'],
'expires':300}}
self.factory = WMFactory('plot_fairy',
'WMCore.HTTPFrontEnd.PlotFairy.Plots')
def plot(self, *args, **kwargs):
input = self.sanitise_input(args, kwargs, 'plot')
plot = self.factory.loadObject(input['type'])
return {'figure': plot(input['data'])}
def doc(self, *args, **kwargs):
input = self.sanitise_input(args, kwargs, 'doc')
plot = self.factory.loadObject(input['type'])
return {'doc': plot.doc()}
def validate_input(self, input, verb, method):
assert 'type' in input.keys(), "no type provided - what kind of plot do you want?"
if method=='doc':
return input
valid_data = {}
assert 'data' in input.keys() or 'url' in input.keys(), "neither data nor url provided - please provide at least one"
if 'url' in input.keys():
match = URL_REGEX.match(input['url'])
assert match != None, "`%s' is not a valid URL" % input['url']
host = match.group(1) + '://' + match.group(3)
if match.group(4):
host += match.group(4)
uri = match.group(5)
result,status,reason,fromcache = JSONRequests(host).get(uri)
valid_data.update(result)
if 'data' in input.keys():
valid_data.update(json.loads(urllib.unquote(input['data'])))
return {'data':valid_data,'type':input['type']}
def __init__(self, config, dbsconfig=None):
"""
Initialise class members
"""
myThread = threading.currentThread()
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
self.uploadToDBS = factory.loadObject("UploadToDBS")
addFactory = WMFactory("dbsBuffer",
"WMComponent.DBSBuffer.Database.Interface")
self.addToBuffer = addFactory.loadObject("AddToBuffer")
# Set DBSInterface
self.dbsInterface = DBSInterface(config=config)
# Set DAOs
self.setBlock = self.bufferFactory(classname="DBSBufferFiles.SetBlock")
self.setStatus = self.bufferFactory(
classname="DBSBufferFiles.SetStatus")
# Set config parameters
self.maxBlockFiles = self.config.DBSInterface.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSInterface.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSInterface.DBSBlockMaxSize
self.doMigration = getattr(self.config.DBSInterface,
'doGlobalMigration', True)
logging.debug(
"Initializing with maxFiles %i, maxTime %i, maxSize %i" %
(self.maxBlockFiles, self.maxBlockTime, self.maxBlockSize))
if dbsconfig == None:
self.dbsconfig = config
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName="DBSUpload")
return
def __init__(self, config, dbsconfig = None):
"""
Initialise class members
"""
myThread = threading.currentThread()
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
self.uploadToDBS = factory.loadObject("UploadToDBS")
addFactory = WMFactory("dbsBuffer",
"WMComponent.DBSBuffer.Database.Interface")
self.addToBuffer = addFactory.loadObject("AddToBuffer")
# Set DBSInterface
self.dbsInterface = DBSInterface(config = config)
# Set DAOs
self.setBlock = self.bufferFactory(classname = "DBSBufferFiles.SetBlock")
self.setStatus = self.bufferFactory(classname = "DBSBufferFiles.SetStatus")
# Set config parameters
self.maxBlockFiles = self.config.DBSInterface.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSInterface.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSInterface.DBSBlockMaxSize
self.doMigration = getattr(self.config.DBSInterface, 'doGlobalMigration', True)
logging.debug("Initializing with maxFiles %i, maxTime %i, maxSize %i" % (self.maxBlockFiles,
self.maxBlockTime,
self.maxBlockSize))
if dbsconfig == None:
self.dbsconfig = config
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "DBSUpload")
return
def __init__(self, options, logger, defaults):
self.defaults = defaults
self.logger = logger
# Format the site name
self.site = options.site.lower()
self.node = options.site.replace('_Buffer', '_MSS')
if not self.node.endswith('_MSS'):
self.node += '_MSS'
# Connect to local couch
self.localcouch = CouchServer(options.localdb)
# Attempt to load the configuration
self.configdb = self.localcouch.connectDatabase('%s_configuration' % self.site)
self.load_config()
# Now update configuration with command line parameters or defaults
for key in defaults:
storeName = defaults[key]['opts']['dest']
# Overwrite those options passed on the command line
cmdOption = getattr(options, storeName)
if cmdOption:
print "Updating config", storeName, cmdOption
self.config[storeName] = cmdOption
# Use defaults for any options which remain unset
if not self.config.has_key(storeName):
print "Using default config", storeName, defaults[key]['opts']['default']
self.config[storeName] = defaults[key]['opts']['default']
# Check waittime is sensible
if self.config['waittime'] < 31:
self.config['waittime'] = 31
# Save the active configuration
self.save_config()
# Finish up remote DB connection, and other config
self.logger.info('local databases: %s' % self.localcouch)
#Create our stager
factory = WMFactory('stager_factory', 'StageManager.Agents.Stagers')
queuedb = self.localcouch.connectDatabase('%s_stagequeue' % self.site)
statsdb = self.localcouch.connectDatabase('%s_statistics' % self.site)
requestdb = self.localcouch.connectDatabase('%s_requests' % self.site)
# Parse stager options
sopts = self.config['stageroptions'].split(",")
def OptFilter(opt):
return opt.find("=") > 0
sopts = filter(OptFilter, sopts)
stagerOptions = {}
for sopt in sopts:
tokens = sopt.split("=")
stagerOptions[tokens[0]] = tokens[1]
self.stager = factory.loadObject(self.config['stagerplugin'],
args=[queuedb, statsdb, self.configdb, requestdb,
stagerOptions, self.logger, self.site],
listFlag = True)
def preInitialization(self):
"""
__preInitialization__
"""
if not hasattr(self.config.PilotManagerComponent, "pilotManagerHandler"):
self.config.PilotManagerComponent.pilotManagerHandler = 'PilotManager.Handler.PilotManagerHandler'
factory = WMFactory('PilotManager');
self.messages["NewPilotJob"] = factory.loadObject(\
self.config.PilotManagerComponent.pilotManagerHandler, self)
self.messages["SubmitPilotJob"] = factory.loadObject(\
self.config.PilotManagerComponent.pilotManagerHandler, self)
#create pilot tar file
self.createPilotTar()
def setSchema(self, modules = [], params = None):
"""
Creates the schema in the database based on the modules
input.
This method needs to have been preceded by the
setDatabaseConnection.
"""
myThread = threading.currentThread()
parameters = None
flag = False
#Set up for typical DBCreator format: logger, dbi, params
if params != None:
parameters = [None, None, params]
flag = True
myThread.transaction.begin()
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
create = factory.loadObject("Create", args = parameters, listFlag = flag)
createworked = create.execute(conn = myThread.transaction.conn,
transaction = myThread.transaction)
if createworked:
logging.debug("Tables for "+ factoryName + " created")
else:
logging.debug("Tables " + factoryName + " could not be created.")
myThread.transaction.commit()
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
#
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
logging.info("Setting the replication to central monitor ...")
self.localSummaryCouchDB.replicate(self.config.AnalyticsDataCollector.centralWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject(classname="Feeder", getFromCache=False)
feeder(fileset)
except Exception as e:
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info(traceback.format_exc())
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject( classname = "Feeder", getFromCache = False )
feeder(fileset)
except Exception as e :
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info( traceback.format_exc() )
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def testF_DBSUploadQueueSizeCheckForAlerts(self):
"""
Test will not trigger a real alert being sent unless doing some
mocking of the methods used during DBSUploadPoller.algorithm() ->
DBSUploadPoller.uploadBlocks() method.
As done here, it probably can't be deterministic, yet the feature
shall be checked.
"""
sizeLevelToTest = 1
myThread = threading.currentThread()
config = self.createConfig()
# threshold / value to check
config.DBSUpload.alertUploadQueueSize = sizeLevelToTest
# without this uploadBlocks method returns immediately
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = sizeLevelToTest + 1
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = "/%s/%s/%s" % (name, name, tier)
# load components that are necessary to check status
# (this seems necessary, else some previous tests started failing)
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
testDBSUpload = DBSUploadPoller(config)
# this is finally where the action (alert) should be triggered from
testDBSUpload.algorithm()
return
class GeneratorFactory(WMFactory):
"""
_GeneratorFactory_
Instantiate a WMFactory instance with the appropriate namespace
"""
def __init__(self):
self.factory = WMFactory(self.__class__.__name__,
"WMCore.JobSplitting.Generators")
def getGenerator(self, generatorName, **options):
"""
_getGenerator_
factory method to return a step template instance based on the
name of the step
"""
args = {}
args.update(options)
try:
return self.factory.loadObject(generatorName, args)
except WMException, wmEx:
msg = "GeneratorFactory Unable to load Object: %s" % generatorName
raise GeneratorFactoryException(msg)
except Exception, ex:
msg = "Error creating object %s in GeneratorFactory:\n" % generatorName
msg += str(ex)
raise GeneratorFactoryException(msg)
def preInitialization(self):
print "JobMaker.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['MakeJob'] = \
factory.loadObject('WMCore.WMSpec.Makers.Handlers.MakeJob', self)
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def __call__(self, parameters):
"""
Handles the event with payload, by sending it to the threadpool.
"""
myThread = threading.currentThread()
#myThread.transaction.begin()
#payload = parameters #I think this is correct; it's all that I've got to go on
factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface")
createWorkArea = factory.loadObject("CreateWorkArea")
jobGroupID = parameters #There should only be one
#We should have, in the payload, exactly one jobGroup ID
createWorkArea.getNewJobGroup(jobGroupID)
createWorkArea.createJobGroupArea()
createWorkArea.createWorkArea()
#For testing purposes only
#DO NOT LEAVE ACTIVE!
#createWorkArea.cleanUpAll()
#myThread.transaction.commit()
return
def __init__(self ):
"""
The constructor creates separate instances
for objects that we do not want to share.
Objects that are used as a thread slave
Inherit from this to ensure they do not
conflict with these methods.
If needed the developer can add additional
objects as it sees fit in the threadslave
class that inherits from this.
"""
self.args = {}
# we also keep a reference to our component (we can
# use this for read only things in the argument list).
# assign this later
self.component = None
#a slave is created in its master thread so we can exploit
#this to get a reference to its dbfactory object.
myThread = threading.currentThread()
self.dbFactory = myThread.dbFactory
# get the procid from the mainthread msg service
# if we use this in testing it might not be there.
self.procid = 0
# we can potentially use mapping from messages to handlers
# to have one thread handling multiple message types.
self.messages = {}
# start loading some objects we need in this thread only
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
def __init__(self):
"""
The constructor creates separate instances
for objects that we do not want to share.
Objects that are used as a thread slave
Inherit from this to ensure they do not
conflict with these methods.
If needed the developer can add additional
objects as it sees fit in the threadslave
class that inherits from this.
"""
self.args = {}
# we also keep a reference to our component (we can
# use this for read only things in the argument list).
# assign this later
self.component = None
#a slave is created in its master thread so we can exploit
#this to get a reference to its dbfactory object.
myThread = threading.currentThread()
self.dbFactory = myThread.dbFactory
# get the procid from the mainthread msg service
# if we use this in testing it might not be there.
self.procid = 0
# we can potentially use mapping from messages to handlers
# to have one thread handling multiple message types.
self.messages = {}
# start loading some objects we need in this thread only
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
class GeneratorFactory(WMFactory):
"""
_GeneratorFactory_
Instantiate a WMFactory instance with the appropriate namespace
"""
def __init__(self):
self.factory = WMFactory(self.__class__.__name__,
"WMCore.JobSplitting.Generators")
def getGenerator(self, generatorName, **options):
"""
_getGenerator_
factory method to return a step template instance based on the
name of the step
"""
args = {}
args.update(options)
try:
return self.factory.loadObject(generatorName,
args)
except WMException, wmEx:
msg = "GeneratorFactory Unable to load Object: %s" % generatorName
raise GeneratorFactoryException(msg)
except Exception, ex:
msg = "Error creating object %s in GeneratorFactory:\n" % generatorName
msg += str(ex)
raise GeneratorFactoryException(msg)
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
"WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def testF_DBSUploadQueueSizeCheckForAlerts(self):
"""
Test will not trigger a real alert being sent unless doing some
mocking of the methods used during DBSUploadPoller.algorithm() ->
DBSUploadPoller.uploadBlocks() method.
As done here, it probably can't be deterministic, yet the feature
shall be checked.
"""
sizeLevelToTest = 1
myThread = threading.currentThread()
config = self.createConfig()
# threshold / value to check
config.DBSUpload.alertUploadQueueSize = sizeLevelToTest
# without this uploadBlocks method returns immediately
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = sizeLevelToTest + 1
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# load components that are necessary to check status
# (this seems necessary, else some previous tests started failing)
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config = config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef = True)
testDBSUpload = DBSUploadPoller(config)
# this is finally where the action (alert) should be triggered from
testDBSUpload.algorithm()
return
def initializeSchema(self, nameSpace, modules = []):
"""
Sometimes you need to initialize the schema before
starting the program. This methods lets you pass
modules that have an execute method which contains
arbitrary sql statements.
"""
myThread = threading.currentThread()
# filter out unique modules
myThread.transaction.begin()
factory = WMFactory(nameSpace, nameSpace + "." + myThread.dialect)
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
create = factory.loadObject(factoryName)
create.logger = logging
create.dbi = myThread.dbi
createworked = create.execute(conn = myThread.transaction.conn)
if createworked:
logging.debug("SQL for "+ factoryName + " executed")
else:
logging.debug("SQL " + factoryName + \
" could not be executed, already exists?")
myThread.transaction.commit()
def preInitialization(self):
print "DBSBuffer.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['JobSuccess'] = \
factory.loadObject(self.config.DBSBuffer.jobSuccessHandler, self)
def __call__(self, parameters):
"""
Handles the event with payload, by sending it to the threadpool.
"""
myThread = threading.currentThread()
#myThread.transaction.begin()
#payload = parameters #I think this is correct; it's all that I've got to go on
factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface")
createWorkArea=factory.loadObject("CreateWorkArea")
jobGroupID = parameters #There should only be one
#We should have, in the payload, exactly one jobGroup ID
createWorkArea.getNewJobGroup(jobGroupID)
createWorkArea.createJobGroupArea()
createWorkArea.createWorkArea()
#For testing purposes only
#DO NOT LEAVE ACTIVE!
#createWorkArea.cleanUpAll()
#myThread.transaction.commit()
return
def preInitialization(self):
print "JobMaker.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['MakeJob'] = \
factory.loadObject('WMCore.WMSpec.Makers.Handlers.MakeJob', self)
def setSchema(self, modules=None, params=None):
"""
Creates the schema in the database based on the modules
input.
This method needs to have been preceded by the
setDatabaseConnection.
"""
modules = modules or []
myThread = threading.currentThread()
parameters = None
flag = False
# Set up for typical DBCreator format: logger, dbi, params
if params != None:
parameters = [None, None, params]
flag = True
myThread.transaction.begin()
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
create = factory.loadObject("Create", args=parameters, listFlag=flag)
createworked = create.execute(conn=myThread.transaction.conn,
transaction=myThread.transaction)
if createworked:
logging.debug("Tables for " + factoryName + " created")
else:
logging.debug("Tables " + factoryName + " could not be created.")
myThread.transaction.commit()
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"):
pluginFactory = WMFactory("specArgs", loc)
alteredClassName = "%sWorkloadFactory" % request_type
spec = pluginFactory.loadObject(classname=request_type, alteredClassName=alteredClassName)
specArgs = spec.getWorkloadArguments()
result = create_json_template_spec(specArgs)
return result
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"):
pluginFactory = WMFactory("specArgs", loc)
alteredClassName = "%sWorkloadFactory" % request_type
spec = pluginFactory.loadObject(classname = request_type, alteredClassName = alteredClassName)
specArgs = spec.getWorkloadArguments()
result = create_json_template_spec(specArgs)
return result
class LFNSourceDuplicator(BaseWorkerThread):
"""
_LFNSourceDuplicator_
Load plugin to get the result of the source database query.
Duplicates the result got into the local database.
"""
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.AsyncTransfer
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
# Asynch db
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
return
def algorithm(self, parameters = None):
"""
_algorithm_
Load the plugin of a db source which load its view.
Duplicates the results got from the plugin in Async database.
"""
self.logger.debug('Duplication algorithm begins')
try:
duplicator = self.factory.loadObject(self.config.pluginName, args = [self.config, self.logger], getFromCache = True, listFlag = True)
except ImportError,e :
msg = "plugin \'%s\' unknown" % self.config.pluginName
self.logger.info(msg)
self.logger.info(e)
for doc in duplicator():
self.db.queue(doc, True, ['AsyncTransfer/ftscp'] )
self.logger.debug("doc queued %s" %doc)
self.db.commit(viewlist=['AsyncTransfer/ftscp'])
self.logger.debug('Duplication done')
return
def __init__(self, config):
self.config = config # Saves the config because Page needs it
self.encoder = json.JSONEncoder()
# The full URL of this openid server
if hasattr(config, 'server_url'):
self.base_path = '%s/' % config.server_url
else:
self.base_path = '%s/' % cherrypy.url()
# Now instantiates the OpenID Store (used for nonces and associations)
if config.store == 'filestore':
store = FileOpenIDStore(config.store_path)
elif config.store == 'dbstore':
store = DBOidStore(config.store_source)
else:
store = MemoryStore()
# Now instantiates the OpenID server
self.openid = server.Server(store, self.base_path + 'openidserver')
# Record how to get user details out...
factory = WMFactory('oid_factory')
self.userstore = factory.loadObject(config.users_store.object,
config.users_store)
# List of remote sites allowed to request auth/authz to this server
self.truststore = factory.loadObject(config.trust_store.object,
config.trust_store)
# This is a thread safe object to display web pages/forms related to
# the oid server. I put it on a separate class to avoid blowing
# the main class with html code.
self.webpages = OidServerWebPages()
# *** Should enable cherrypy session somewhere... sorry
cherrypy.config.update({
'tools.sessions.on': True,
'tools.sessions.storage_type': 'ram',
#'tools.sessions.storage_type': "file",
#'tools.sessions.storage_path': "/tmp/oidserver-sessions",
'tools.sessions.timeout': 60,
'tools.sessions.name': 'oidserver_sid'
})
def getDBinstance(config, namespace, name):
if config.backend.lower() == 'mysql':
backend = 'MySQL'
elif config.backend.lower() == 'oracle':
backend = 'Oracle'
#factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name = name, namespace = 'Databases.%s.%s.%s' % (namespace,backend,name))
return factory.loadObject( name )
def getDBinstance(config, namespace, name):
if config.backend.lower() == 'mysql':
backend = 'MySQL'
elif config.backend.lower() == 'oracle':
backend = 'Oracle'
#factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name = name, namespace = 'Databases.%s.%s.%s' % (namespace,backend,name))
return factory.loadObject( name )
def __init__(self, config):
self.config = config # Saves the config because Page needs it
self.encoder = json.JSONEncoder()
# The full URL of this openid server
if hasattr(config,'server_url'):
self.base_path = '%s/' % config.server_url
else:
self.base_path = '%s/' % cherrypy.url()
# Now instantiates the OpenID Store (used for nonces and associations)
if config.store == 'filestore':
store = FileOpenIDStore(config.store_path)
elif config.store == 'dbstore':
store = DBOidStore(config.store_source)
else:
store = MemoryStore()
# Now instantiates the OpenID server
self.openid = server.Server(store, self.base_path + 'openidserver')
# Record how to get user details out...
factory = WMFactory('oid_factory')
self.userstore = factory.loadObject(config.users_store.object,
config.users_store)
# List of remote sites allowed to request auth/authz to this server
self.truststore = factory.loadObject(config.trust_store.object,
config.trust_store)
# This is a thread safe object to display web pages/forms related to
# the oid server. I put it on a separate class to avoid blowing
# the main class with html code.
self.webpages = OidServerWebPages()
# *** Should enable cherrypy session somewhere... sorry
cherrypy.config.update({'tools.sessions.on': True,
'tools.sessions.storage_type': 'ram',
#'tools.sessions.storage_type': "file",
#'tools.sessions.storage_path': "/tmp/oidserver-sessions",
'tools.sessions.timeout': 60,
'tools.sessions.name': 'oidserver_sid'})
def updateDSFileCount(self, dataset):
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database."+ \
myThread.dialect)
newDS = factory.loadObject("UpdateDSFileCount")
newDS.execute(dataset=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def updateDSFileCount(self, dataset):
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database."+ \
myThread.dialect)
newDS = factory.loadObject("UpdateDSFileCount")
newDS.execute(dataset=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def __init__(self, params = None):
"""
Constructor. The 'params' argument can be used as a dict for any
parameter (if needed). Basic things can be obtained from currentThread.
The required params are as follow:
downloadBaseUrl, sandboxBasePath, specBasePath, matcherPlugin
"""
myThread = threading.currentThread()
# factory = WMFactory("default", \
# "TQComp.Database."+myThread.dialect)
factory = WMFactory("default")
# self.queries = factory.loadObject("Queries")
self.queries = factory.loadObject("TQComp.Database."\
+myThread.dialect+".Queries")
self.logger = myThread.logger
required = ["downloadBaseUrl", "sandboxBasePath", "specBasePath",\
"uploadBaseUrl", "matcherPlugin", "maxThreads"]
for param in required:
if not param in params:
messg = "GetTaskHandler object requires params['%s']" % param
# TODO: What number?
numb = 0
raise WMException(messg, numb)
self.downloadBaseUrl = params["downloadBaseUrl"]
self.sandboxBasePath = params["sandboxBasePath"]
self.specBasePath = params["specBasePath"]
self.uploadBaseUrl = params["uploadBaseUrl"]
self.maxThreads = params["maxThreads"]
try:
self.matcher = factory.loadObject(params["matcherPlugin"],\
{'queries': self.queries, 'logger': self.logger})
except:
ttype, val, tb = sys.exc_info()
# myThread.transaction.rollback()
messg = 'Could not load matcher plugin, due to: %s - %s '% (ttype, val)
# TODO: What number?
numb = 0
raise WMException(messg, numb)
class RetryManagerPoller(BaseWorkerThread):
"""
_RetryManagerPoller_
Polls for Jobs in CoolOff State and attempts to retry them
based on the requirements in the selected plugin
"""
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
pluginPath = getattr(self.config.RetryManager, "pluginPath",
"WMComponent.RetryManager.PlugIns")
self.pluginFactory = WMFactory("plugins", pluginPath)
self.changeState = ChangeState(self.config)
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
# initialize the alert framework (if available) (self.sendAlert())
self.initAlerts(compName="RetryManager")
# get needed plugins
self.plugins = {}
self.typePluginsAssoc = getattr(self.config.RetryManager, 'plugins',
{})
self.typePluginsAssoc.setdefault('default', 'DefaultRetryAlgo')
for pluginName in self.typePluginsAssoc.values():
try:
plugin = self.pluginFactory.loadObject(classname=pluginName,
args=config)
self.plugins[pluginName] = plugin
except Exception, ex:
msg = "Error loading plugin %s on path %s\n" % (pluginName,
pluginPath)
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
raise RetryManagerException(msg)
return
class RetryManagerPoller(BaseWorkerThread):
"""
_RetryManagerPoller_
Polls for Jobs in CoolOff State and attempts to retry them
based on the requirements in the selected plugin
"""
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
pluginPath = getattr(self.config.RetryManager, "pluginPath",
"WMComponent.RetryManager.PlugIns")
self.pluginFactory = WMFactory("plugins", pluginPath)
self.changeState = ChangeState(self.config)
self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")
# initialize the alert framework (if available) (self.sendAlert())
self.initAlerts(compName = "RetryManager")
# get needed plugins
self.plugins = {}
self.typePluginsAssoc = getattr(self.config.RetryManager, 'plugins', {})
self.typePluginsAssoc.setdefault('default', 'DefaultRetryAlgo')
for pluginName in self.typePluginsAssoc.values():
try:
plugin = self.pluginFactory.loadObject(classname = pluginName,
args = config)
self.plugins[pluginName] = plugin
except Exception, ex:
msg = "Error loading plugin %s on path %s\n" % (pluginName, pluginPath)
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
raise RetryManagerException(msg)
return
class GeneratorFactory(WMFactory):
"""
_GeneratorFactory_
Instantiate a WMFactory instance with the appropriate namespace
"""
def __init__(self):
self.factory = WMFactory(self.__class__.__name__,
"WMCore.JobSplitting.Generators")
def getGenerator(self, generatorName, **options):
"""
_getGenerator_
factory method to return a step template instance based on the
name of the step
"""
args = {}
args.update(options)
try:
return self.factory.loadObject(generatorName,
args)
except WMException as wmEx:
msg = "GeneratorFactory Unable to load Object: %s" % generatorName
raise GeneratorFactoryException(msg)
except Exception as ex:
msg = "Error creating object %s in GeneratorFactory:\n" % generatorName
msg += str(ex)
raise GeneratorFactoryException(msg)
return
def makeGenerators(self, wmTaskHelper):
"""
_makeGenerators_
Util function to build the set of generators defined
by the task provided
"""
result = []
for generator in wmTaskHelper.listGenerators():
genInstance = self.getGenerator(generator,
wmTaskHelper,
**wmTaskHelper.getGeneratorSettings(generator)
)
result.append(genInstance)
return result
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
myThread = threading.currentThread()
factory = WMFactory("WMBS", "WMCore.WMBS")
destroy = factory.loadObject(myThread.dialect + ".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn=myThread.transaction.conn)
if not destroyworked:
raise Exception("Could not complete WMBS tear down.")
myThread.transaction.commit()
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
myThread = threading.currentThread()
factory = WMFactory("WMBS", "WMCore.WMBS")
destroy = factory.loadObject(myThread.dialect + ".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn=myThread.transaction.conn)
if not destroyworked:
raise Exception("Could not complete WMBS tear down.")
myThread.transaction.commit()
def setBlockStatus(self, block, locations, openStatus=0, time=0):
"""
_setBlockStatus_
Adds information about the block from DBSWriter to the database
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
newDS = factory.loadObject("SetBlockStatus")
newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \
conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def setBlockStatus(self, block, locations, openStatus = 0, time = 0):
"""
_setBlockStatus_
Adds information about the block from DBSWriter to the database
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
newDS = factory.loadObject("SetBlockStatus")
newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \
conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
class GeneratorFactory(WMFactory):
"""
_GeneratorFactory_
Instantiate a WMFactory instance with the appropriate namespace
"""
def __init__(self):
self.factory = WMFactory(self.__class__.__name__,
"WMCore.JobSplitting.Generators")
def getGenerator(self, generatorName, **options):
"""
_getGenerator_
factory method to return a step template instance based on the
name of the step
"""
args = {}
args.update(options)
try:
return self.factory.loadObject(generatorName, args)
except WMException as wmEx:
msg = "GeneratorFactory Unable to load Object: %s" % generatorName
raise GeneratorFactoryException(msg)
except Exception as ex:
msg = "Error creating object %s in GeneratorFactory:\n" % generatorName
msg += str(ex)
raise GeneratorFactoryException(msg)
return
def makeGenerators(self, wmTaskHelper):
"""
_makeGenerators_
Util function to build the set of generators defined
by the task provided
"""
result = []
for generator in wmTaskHelper.listGenerators():
genInstance = self.getGenerator(
generator, wmTaskHelper,
**wmTaskHelper.getGeneratorSettings(generator))
result.append(genInstance)
return result
def initInThread(self):
"""
Called during thread initialization. Loads the
backend for this instance.
"""
# make sure you instantiate the super class method.
ThreadSlave.initInThread(self)
# load queries for backend.
myThread = threading.currentThread()
factory = WMFactory("default", \
"TQComp.Database."+myThread.dialect)
# make sure you do not overload attributes
# defined in your parent class
self.queries = factory.loadObject("Queries")
logging.debug("ParentSlave initialized")
def preInitialization(self):
if not hasattr(self.config.PilotMonitorComponent, "pilotMonitorHandler"):
self.config.PilotMonitorComponent.pilotMonitorHandler = \
'PilotMonitor.Handler.PilotMonitorHandler'
factory = WMFactory("PilotMonitor");
self.messages["MonitorPilots"] = factory.loadObject(\
self.config.PilotMonitorComponent.pilotMonitorHandler, self)
tqconfig = Configuration.loadConfigurationFile( \
self.config.PilotMonitorComponent.TQConfig )
myThread = threading.currentThread()
self.logger = myThread.logger
self.logger.debug( tqconfig)
self.tqStateApi = TQStateApi(self.logger, tqconfig, None)
def clearDatabase(self, modules = []):
"""
Enables clearing particular tables in the database
Associated to modules. Note this only works if there
is the module has a Destroy class. Beware that this
might not work if there are table dependencies.
"""
myThread = threading.currentThread()
for module in modules:
factory = WMFactory("clear", module)
destroy = factory.loadObject(myThread.dialect+".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn = myThread.transaction.conn)
if not destroyworked:
raise Exception(module +" tables could not be destroyed")
myThread.transaction.commit()
del factory
def findAlgos(self, dataset):
"""
Call to findAlgos
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
findAlgos = factory.loadObject("FindAlgos")
# Add the file to the buffer (API Call)
#results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction())
results = findAlgos.execute(datasetInfo=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return results
def testE_NoMigration(self):
"""
_NoMigration_
Test the DBSUpload system with no global migration
"""
myThread = threading.currentThread()
config = self.createConfig()
self.injectWorkflow(MaxWaitTime=3)
config.DBSInterface.doGlobalMigration = False
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config=config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('Open', )])
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1"
)[0].fetchall()
for r in result:
self.assertEqual(r[0], 'GLOBAL')
return
def runDas(self, func, data, expires):
"""
Run a query and produce a dictionary for DAS formatting
"""
start_time = time.time()
results = func(self, data)
call_time = time.time() - start_time
res_expire = make_timestamp(expires)
if isinstance(results, list):
if len(results) > 0:
row = results[0]
else:
row = None
else:
row = results
if isinstance(row, str):
row = '"%s"' % row
try:
factory = WMFactory('webtools_factory')
obj = factory.loadObject(self.config.model.object, self.config)
res_version = obj.version
except Exception:
res_version = 'unknown'
keyhash = hashlib.md5()
if not isinstance(results, (str, bytes)):
keyhash.update(encodeUnicodeToBytesConditional(str(results), condition=PY3))
else:
keyhash.update(encodeUnicodeToBytesConditional(results, condition=PY3))
res_checksum = keyhash.hexdigest()
dasdata = {'application': '%s.%s' % (self.config.application, func.__name__),
'request_timestamp': start_time,
'request_url': request.base + request.path_info + '?' + request.query_string,
'request_method': request.method,
'request_params': request.params,
'response_version': res_version,
'response_expires': res_expire,
'response_checksum': res_checksum,
'request_call': func.__name__,
'call_time': call_time,
'results': results,
}
return dasdata
def findAlgos(self, dataset):
"""
Call to findAlgos
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
findAlgos = factory.loadObject("FindAlgos")
# Add the file to the buffer (API Call)
#results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction())
results = findAlgos.execute(datasetInfo=dataset,
conn=self.getDBConn(),
transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return results
def testD(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
self.tempDir = self.testInit.generateWorkDir(config)
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent2")
os.makedirs(config.TestComponent.componentDir)
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent=True)
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir,
"Daemon.xml")
details = Details(daemonFile)
print('Is component alive: ' + str(details.isAlive()))
# create msgService to send stop message.
myThread = threading.currentThread()
factory = WMFactory("msgService", "WMCore.MsgService." + \
myThread.dialect)
myThread.transaction = Transaction(myThread.dbi)
msgService = factory.loadObject("MsgService")
msgService.registerAs("HarnessTest")
myThread.transaction.commit()
print(
'Publish a stop message to test if the component shutsdown gracefully'
)
myThread.transaction.begin()
msg = {'name': 'Stop', 'payload': ''}
msgService.publish(msg)
myThread.transaction.commit()
msgService.finish()
while details.isAlive():
print('Component has not received stop message')
time.sleep(2)
print('Daemon shutdown gracefully')
def runDas(self, func, data, expires):
"""
Run a query and produce a dictionary for DAS formatting
"""
start_time = time.time()
results = func(self, data)
call_time = time.time() - start_time
res_expire = make_timestamp(expires)
if type(results) is types.ListType:
if len(results) > 0:
row = results[0]
else:
row = None
else:
row = results
if type(row) is types.StringType:
row = '"%s"' % row
try:
factory = WMFactory('webtools_factory')
object = factory.loadObject(self.config.model.object, self.config)
res_version = object.version
except:
res_version = 'unknown'
keyhash = hashlib.md5()
keyhash.update(str(results))
res_checksum = keyhash.hexdigest()
dasdata = {'application':'%s.%s' % (self.config.application, func.__name__),
'request_timestamp': start_time,
'request_url': request.base + request.path_info + '?' + \
request.query_string,
'request_method' : request.method,
'request_params' : request.params,
'response_version': res_version,
'response_expires': res_expire,
'response_checksum': res_checksum,
'request_call': func.__name__,
'call_time': call_time,
'results': results,
}
return dasdata
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def __init__(self, config):
myThread = threading.currentThread()
self.config = config
self.dbsurl = self.config.DBSUpload.dbsurl
self.dbsversion = self.config.DBSUpload.dbsversion
self.uploadFileMax = self.config.DBSUpload.uploadFileMax
self.DBSMaxFiles = self.config.DBSUpload.DBSMaxFiles
self.DBSMaxSize = self.config.DBSUpload.DBSMaxSize
self.DBSBlockTimeout = self.config.DBSUpload.DBSBlockTimeout
self.dbswriter = DBSWriter(self.dbsurl, level='ERROR', user='******', version=self.dbsversion, \
globalDBSUrl = self.config.DBSUpload.globalDBSUrl, \
globalVersion = self.config.DBSUpload.globalDBSVer)
self.daoFactory = DAOFactory(package="WMComponent.DBSBuffer.Database",
logger=logging,
dbinterface=myThread.dbi)
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
self.dbinterface = factory.loadObject("UploadToDBS")
def preInitialization(self):
"""
Add required worker modules to work threads
"""
# in case nothing was configured we have a fallback.
if not hasattr(self.config.FeederManager, "addDatasetWatch"):
logging.warning("Using default AddDatasetWatch handler")
self.config.FeederManager.addDatasetWatchHandler = \
'WMComponent.FeederManager.Handler.DefaultAddDatasetWatch'
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['AddDatasetWatch'] = \
factory.loadObject(\
self.config.FeederManager.addDatasetWatchHandler, self)
myThread = threading.currentThread()
myThread.runningFeedersLock = threading.Lock()
myThread.runningFeeders = {}
pollInterval = self.config.FeederManager.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(FeederManagerPoller(), \
pollInterval)
class Watchdog(threading.Thread):
"""
Watchdog
It's like a watchdog! It watches things, and then it barks at them, curls up, and goes to sleep
Or possibly monitors them and records what they do. It's a bit of a crapshoot at this point.
"""
def __init__(self, logPath=None, config=None):
threading.Thread.__init__(self)
self.doMonitoring = True
self._Finished = threading.Event()
self._EndOfJob = threading.Event()
self._NewTask = threading.Event()
self._JobKilled = threading.Event()
self._RunUpdate = threading.Event()
self._Interval = 120.0
self._Monitors = []
# Right now we join this, because we don't know
# Where we'll be when we need this.
self.logPath = os.path.join(os.getcwd(), logPath)
self.factory = WMFactory(self.__class__.__name__,
"WMCore.WMRuntime.Monitors")
def setupMonitors(self, task, wmbsJob):
logging.info("In Watchdog.setupMonitors")
if not hasattr(task.data, 'watchdog'):
msg = "Could not find watchdog in spec"
logging.error(msg)
# I don't think this is necessarily fatal
return
if not hasattr(task.data.watchdog, 'monitors'):
msg = "Watchdog has no monitors"
logging.error(msg)
# Probably not fatal either
return
if hasattr(task.data.watchdog, 'interval'):
# Set the interval off the config
self.setInterval(task.data.watchdog.interval)
for monitor in task.data.watchdog.monitors:
msg = "Initializing monitor %s" % monitor
logging.info(msg)
mon = self.loadMonitor(monitor)
args = {}
if hasattr(task.data.watchdog, monitor):
# This should be a config section
monitorArgs = getattr(task.data.watchdog, monitor)
args = monitorArgs.dictionary_()
if monitor == 'PerformanceMonitor' and args:
# Apply tweaks to PerformanceMonitor only.
# Scale resources according to the HTCondor runtime environment.
origCores = 1
for stepName in task.listAllStepNames():
sh = task.getStepHelper(stepName)
origCores = max(origCores, sh.getNumberOfCores())
resources = {'cores': origCores}
origMaxPSS = args.get('maxPSS', args.get('maxRSS'))
if origMaxPSS:
resources['memory'] = origMaxPSS
# Actually parses the HTCondor runtime
resizeResources(resources)
# We decided to only touch Watchdog settings if the number of cores changed.
# (even if this means the watchdog memory is wrong for a slot this size).
changedCores = origCores != resources['cores']
# If we did base maxPSS off the memory in the HTCondor slot, subtract a bit
# off the top so watchdog triggers before HTCondor does.
# Add the new number of cores to the args such that PerformanceMonitor can see it
args['cores'] = resources['cores']
if changedCores:
if origMaxPSS:
args['maxPSS'] = resources['memory'] - 50
logging.info("Watchdog modified: %s. Final settings:",
changedCores)
for k, v in viewitems(args):
logging.info(" %s: %r", k, v)
# Actually initialize the monitor variables
mon.initMonitor(task=task,
job=wmbsJob,
logPath=self.logPath,
args=args)
self._Monitors.append(mon)
return
def loadMonitor(self, monitorName):
try:
return self.factory.loadObject(monitorName)
except WMException:
msg = "WatchdogFactory Unable to load Object: %s" % monitorName
logging.error(msg)
raise WatchdogException(msg)
except Exception as ex:
msg = "Error creating object %s in WatchdogFactory:\n" % monitorName
msg += str(ex)
logging.error(msg)
raise WatchdogException(msg)
def setInterval(self, interval):
"""
Set the monitor interval
"""
logging.info("Set Watchdog interval to %s", interval)
self._Interval = interval
def disableMonitoring(self):
"""
_disableMonitoring_
Turn off active monitoring (periodicUpdate calls are disabled,
event driven monitoring still occurs
"""
self.doMonitoring = False
def shutdown(self):
"""
Shutdown the monitor.
"""
logging.info("MonitorState: Shutdown called")
self._Finished.set()
return
# //=========notify Methods called by the ExecutionManager====
# //
# // Start notification from the exe thread, this starts the
# //periodic updates of the monitor thread
# //
# //
def notifyJobStart(self, task):
"""
Start the job.
"""
logging.info("MonitorThread: JobStarted")
if self.doMonitoring:
self.setDaemon(1)
self.start()
for monitor in self._Monitors:
try:
monitor.jobStart(task)
except Exception as ex:
msg = "Error in notifyJobStart for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise WatchdogException(msg)
return
# //
# // notify Monitors of new task start up
# //
def notifyStepStart(self, step):
"""
notify Monitors of new task start up.
"""
self._RunUpdate.set()
for monitor in self._Monitors:
try:
monitor.stepStart(step)
except Exception as ex:
msg = "Error in notifyTaskStart for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise WatchdogException(msg)
return
# //
# // notify Monitors of task completion
# //
def notifyStepEnd(self, step, exitCode=0, stepReport=None):
"""
notify Monitors of task completion.
"""
self._RunUpdate.clear()
for monitor in self._Monitors:
try:
monitor.stepEnd(step=step, stepReport=stepReport)
except Exception as ex:
msg = "Error in notifyTaskEnd for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise WatchdogException(msg)
# print "Task Ended: %s with Exit Code:%s" % (task, exitCode)
# self._MonMgr.taskEnd(task, exitCode)
return
# //
# // notify monitors of Job Completion, stops the periodic
# // updating
def notifyJobEnd(self, task):
"""
notify monitors of Job Completion, stops the periodic
updating.
"""
logging.info("MonitorThread: JobEnded")
for monitor in self._Monitors:
try:
monitor.jobEnd(task)
except Exception as ex:
msg = "Error in notifyJobEnd for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
raise WatchdogException(msg)
# self._MonMgr.jobEnd()
self.shutdown()
return
# //
# // Interrupt Notifiers
# // Job has been killed
def notifyKillJob(self):
"""
Interrupt Notifiers, Job has been killed.
"""
logging.info("MonitorThread: JobKilled")
for monitor in self._Monitors:
try:
monitor.jobKilled()
except Exception as ex:
msg = "Error in notifyKillJob for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise WatchdogException(msg)
# self._MonMgr.jobKilled()
self.shutdown()
# //
# // Task has been killed
# //
def notifyKillStep(self, step=None):
"""
Task has been killed.
"""
logging.info("MonitorThread: TaskKilled")
for monitor in self._Monitors:
try:
monitor.stepKilled(step)
except Exception as ex:
msg = "Error in notifyKillTask for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
logging.error(msg)
raise WatchdogException(msg)
# self._MonMgr.taskKilled()
# //
# // Override Thread.run() to do the periodic update
# // of the MonitorState object and dispatch it to the monitors
def run(self):
"""
Override Thread.run() to do the periodic update
of the MonitorState object and dispatch it to the monitors
"""
while True:
# //
# // shutdown signal
# //
if self._Finished.isSet():
return
# //
# // Update State information only during a running task
# //
if self._RunUpdate.isSet():
for monitor in self._Monitors:
try:
monitor.periodicUpdate()
except Exception as ex:
msg = "Error in periodicUpdate for monitor class %s in Watchdog:\n" % monitor.__class__
msg += str(ex)
msg += str(traceback.format_exc())
msg += "This is a CRITICAL error because this kills the monitoring.\n"
msg += "Terminate thread and retry.\n"
logging.error(msg)
# raise WatchdogException(msg)
# This one needs to be killed by itself
# since it's run by thread
os.abort()
# self._MonMgr.periodicUpdate()
# time.sleep(self._Interval)
self._Finished.wait(self._Interval)
# //
# // Load Monitor Objects based on Cfg settings passed
# // from Executor
def initMonitorFwk(self, monitorCfg, updatorCfg):
"""
_initMonitorFwk_
Initialise the MonitorMgr object when this method is
called from the Execution thread
Load Monitor Objects based on Cfg settings passed from Executor.
"""
# self._MonMgr.monitorConfig = monitorCfg
# self._MonMgr.updatorConfig = updatorCfg
# self._MonMgr.loadMonitors()
# self._MonMgr.loadUpdators()
return
def testBulkLoad(self):
"""
_testBulkLoad_
Can we load in bulk?
"""
bufferFactory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database.Interface")
addToBuffer = bufferFactory.loadObject("AddToBuffer")
bulkLoad = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID")
testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024,
events = 20)
testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024,
events = 20)
testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024,
events = 20)
testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildA.create()
testFileChildB.create()
testFileChildC.create()
testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"])
testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"])
runSet = set()
runSet.add(Run( 1, *[45]))
runSet.add(Run( 2, *[67, 68]))
testFileChildA.addRunSet(runSet)
testFileChildB.addRunSet(runSet)
testFileChildC.addRunSet(runSet)
testFileChildA.save()
testFileChildB.save()
testFileChildC.save()
setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
binds = [{'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201},
{'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101},
{'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201},
{'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101},
{'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201},
{'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101}]
setCksumAction.execute(bulkList = binds)
testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024,
events = 10)
testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFile.create()
testFileChildA.addParents([testFile["lfn"]])
testFileChildB.addParents([testFile["lfn"]])
testFileChildC.addParents([testFile["lfn"]])
binds = [{'id': testFileChildA.exists()},
{'id': testFileChildB.exists()},
{'id': testFileChildC.exists()}]
listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs = binds)
#print listOfFiles
compareList = ['locations', 'psetHash', 'configContent', 'appName',
'appVer', 'appFam', 'events', 'datasetPath', 'runs']
for f in listOfFiles:
self.assertTrue(f['lfn'] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB",
"/this/is/a/child/lfnC"],
"Unknown file in loaded results")
self.assertEqual(f['checksums'], {'adler32': '201', 'cksum': '101'})
for parent in f['parents']:
self.assertEqual(parent['lfn'], testFile['lfn'])
for key in compareList:
self.assertEqual(f[key], testFileChildA[key])
def testRegistry(self):
logging.debug("Initialize registry")
# somewhere (e.g. during initialization) you create a registry for queries
registry = WMFactory('my_test_registry', WMFactoryTest._repository)
testRegisteredFile = \
registry.loadObject('TestCollection_1.TestRegistryFile1')
parameters = {'par1':'val1', 'par2':'val2'}
result = testRegisteredFile.doSomething(parameters)
self.assertEqual(parameters, result)
# this time from cache
testRegisteredFile = \
registry.loadObject('TestCollection_1.TestRegistryFile2')
parameters = {'par3':'val3', 'par4':'val4'}
result = testRegisteredFile.doSomething(parameters)
self.assertEqual(parameters, result)
testRegisteredFile = \
registry.loadObject('TestCollection_2.TestRegistryFile1')
parameters = {'par5':'val5', 'par6':'val6'}
result = testRegisteredFile.doSomething(parameters)
self.assertEqual(parameters, result)
testRegisteredFile = \
registry.loadObject('TestCollection_2.TestRegistryFile2')
parameters = {'par7':'val7', 'par8':'val8'}
result = testRegisteredFile.doSomething(parameters)
self.assertEqual(parameters, result)
logging.debug("testing cache")
testRegisteredFile = \
registry.loadObject('TestCollection_2.TestRegistryFile2')
testRegisteredFile = \
registry.loadObject('TestCollection_2.TestRegistryFile2')
testRegisteredFile = \
registry.loadObject('TestCollection_2.TestRegistryFile2')
logging.debug("testing multiple instances (no cache)")
testRegisteredFileInst1 = \
registry.loadObject('TestCollection_2.TestRegistryFile2', \
storeInCache = False)
testRegisteredFileInst2 = \
registry.loadObject('TestCollection_2.TestRegistryFile2', \
storeInCache = False)
testRegisteredFileInst3 = \
registry.loadObject('TestCollection_2.TestRegistryFile2', \
storeInCache = False)
parameters = {'par9':'val9', 'par10':'val10'}
result = testRegisteredFileInst1.doSomething(parameters)
self.assertEqual(parameters, result)
parameters = {'par11':'val11', 'par12':'val12'}
result = testRegisteredFileInst2.doSomething(parameters)
self.assertEqual(parameters, result)
parameters = {'par13':'val13', 'par14':'val14'}
result = testRegisteredFileInst3.doSomething(parameters)
self.assertEqual(parameters, result)
logging.debug("Retrieving registry from thread attribute and load objects")
logging.debug("this way all objects in this thread can use the registry")
myThread = threading.currentThread()
registry = myThread.factory['my_test_registry']
parameters = {'par15':'val15', 'par16':'val16'}
testRegisteredFile = \
registry.loadObject('TestCollection_3.TestRegistryFile1', \
args = parameters)
result = testRegisteredFile.doSomething(parameters)
self.assertEqual(parameters, result)
