def setSchema(self, modules = [], params = None):
"""
Creates the schema in the database based on the modules
input.
This method needs to have been preceded by the
setDatabaseConnection.
"""
myThread = threading.currentThread()
parameters = None
flag = False
#Set up for typical DBCreator format: logger, dbi, params
if params != None:
parameters = [None, None, params]
flag = True
myThread.transaction.begin()
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
create = factory.loadObject("Create", args = parameters, listFlag = flag)
createworked = create.execute(conn = myThread.transaction.conn,
transaction = myThread.transaction)
if createworked:
logging.debug("Tables for "+ factoryName + " created")
else:
logging.debug("Tables " + factoryName + " could not be created.")
myThread.transaction.commit()
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
#
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
logging.info("Setting the replication to central monitor ...")
self.localSummaryCouchDB.replicate(self.config.AnalyticsDataCollector.centralWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def preInitialization(self):
print "JobMaker.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['MakeJob'] = \
factory.loadObject('WMCore.WMSpec.Makers.Handlers.MakeJob', self)
def preInitialization(self):
print "DBSBuffer.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['JobSuccess'] = \
factory.loadObject(self.config.DBSBuffer.jobSuccessHandler, self)
def initializeSchema(self, nameSpace, modules = []):
"""
Sometimes you need to initialize the schema before
starting the program. This methods lets you pass
modules that have an execute method which contains
arbitrary sql statements.
"""
myThread = threading.currentThread()
# filter out unique modules
myThread.transaction.begin()
factory = WMFactory(nameSpace, nameSpace + "." + myThread.dialect)
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
create = factory.loadObject(factoryName)
create.logger = logging
create.dbi = myThread.dbi
createworked = create.execute(conn = myThread.transaction.conn)
if createworked:
logging.debug("SQL for "+ factoryName + " executed")
else:
logging.debug("SQL " + factoryName + \
" could not be executed, already exists?")
myThread.transaction.commit()
def preInitialization(self):
print "JobMaker.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['MakeJob'] = \
factory.loadObject('WMCore.WMSpec.Makers.Handlers.MakeJob', self)
def __init__(self, options, logger, defaults):
self.defaults = defaults
self.logger = logger
# Format the site name
self.site = options.site.lower()
self.node = options.site.replace('_Buffer', '_MSS')
if not self.node.endswith('_MSS'):
self.node += '_MSS'
# Connect to local couch
self.localcouch = CouchServer(options.localdb)
# Attempt to load the configuration
self.configdb = self.localcouch.connectDatabase('%s_configuration' % self.site)
self.load_config()
# Now update configuration with command line parameters or defaults
for key in defaults:
storeName = defaults[key]['opts']['dest']
# Overwrite those options passed on the command line
cmdOption = getattr(options, storeName)
if cmdOption:
print "Updating config", storeName, cmdOption
self.config[storeName] = cmdOption
# Use defaults for any options which remain unset
if not self.config.has_key(storeName):
print "Using default config", storeName, defaults[key]['opts']['default']
self.config[storeName] = defaults[key]['opts']['default']
# Check waittime is sensible
if self.config['waittime'] < 31:
self.config['waittime'] = 31
# Save the active configuration
self.save_config()
# Finish up remote DB connection, and other config
self.logger.info('local databases: %s' % self.localcouch)
#Create our stager
factory = WMFactory('stager_factory', 'StageManager.Agents.Stagers')
queuedb = self.localcouch.connectDatabase('%s_stagequeue' % self.site)
statsdb = self.localcouch.connectDatabase('%s_statistics' % self.site)
requestdb = self.localcouch.connectDatabase('%s_requests' % self.site)
# Parse stager options
sopts = self.config['stageroptions'].split(",")
def OptFilter(opt):
return opt.find("=") > 0
sopts = filter(OptFilter, sopts)
stagerOptions = {}
for sopt in sopts:
tokens = sopt.split("=")
stagerOptions[tokens[0]] = tokens[1]
self.stager = factory.loadObject(self.config['stagerplugin'],
args=[queuedb, statsdb, self.configdb, requestdb,
stagerOptions, self.logger, self.site],
listFlag = True)
def __init__(self):
"""
The constructor creates separate instances
for objects that we do not want to share.
Objects that are used as a thread slave
Inherit from this to ensure they do not
conflict with these methods.
If needed the developer can add additional
objects as it sees fit in the threadslave
class that inherits from this.
"""
self.args = {}
# we also keep a reference to our component (we can
# use this for read only things in the argument list).
# assign this later
self.component = None
#a slave is created in its master thread so we can exploit
#this to get a reference to its dbfactory object.
myThread = threading.currentThread()
self.dbFactory = myThread.dbFactory
# get the procid from the mainthread msg service
# if we use this in testing it might not be there.
self.procid = 0
# we can potentially use mapping from messages to handlers
# to have one thread handling multiple message types.
self.messages = {}
# start loading some objects we need in this thread only
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
def testF_DBSUploadQueueSizeCheckForAlerts(self):
"""
Test will not trigger a real alert being sent unless doing some
mocking of the methods used during DBSUploadPoller.algorithm() ->
DBSUploadPoller.uploadBlocks() method.
As done here, it probably can't be deterministic, yet the feature
shall be checked.
"""
sizeLevelToTest = 1
myThread = threading.currentThread()
config = self.createConfig()
# threshold / value to check
config.DBSUpload.alertUploadQueueSize = sizeLevelToTest
# without this uploadBlocks method returns immediately
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = sizeLevelToTest + 1
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = "/%s/%s/%s" % (name, name, tier)
# load components that are necessary to check status
# (this seems necessary, else some previous tests started failing)
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
testDBSUpload = DBSUploadPoller(config)
# this is finally where the action (alert) should be triggered from
testDBSUpload.algorithm()
return
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.AsyncTransfer
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
# Asynch db
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
return
def preInitialization(self):
"""
Add required worker modules to work threads
"""
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['AddWorkflowToManage'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManage", self)
self.messages['RemoveWorkflowFromManagement'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagement", self)
self.messages['AddToWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationList", self)
self.messages['RemoveFromWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationList", self)
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkflowManager.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(WorkflowManagerPoller(), \
pollInterval)
def setSchema(self, modules=None, params=None):
"""
Creates the schema in the database based on the modules
input.
This method needs to have been preceded by the
setDatabaseConnection.
"""
modules = modules or []
myThread = threading.currentThread()
parameters = None
flag = False
# Set up for typical DBCreator format: logger, dbi, params
if params != None:
parameters = [None, None, params]
flag = True
myThread.transaction.begin()
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
create = factory.loadObject("Create", args=parameters, listFlag=flag)
createworked = create.execute(conn=myThread.transaction.conn,
transaction=myThread.transaction)
if createworked:
logging.debug("Tables for " + factoryName + " created")
else:
logging.debug("Tables " + factoryName + " could not be created.")
myThread.transaction.commit()
def testF_DBSUploadQueueSizeCheckForAlerts(self):
"""
Test will not trigger a real alert being sent unless doing some
mocking of the methods used during DBSUploadPoller.algorithm() ->
DBSUploadPoller.uploadBlocks() method.
As done here, it probably can't be deterministic, yet the feature
shall be checked.
"""
sizeLevelToTest = 1
myThread = threading.currentThread()
config = self.createConfig()
# threshold / value to check
config.DBSUpload.alertUploadQueueSize = sizeLevelToTest
# without this uploadBlocks method returns immediately
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = sizeLevelToTest + 1
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# load components that are necessary to check status
# (this seems necessary, else some previous tests started failing)
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config = config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef = True)
testDBSUpload = DBSUploadPoller(config)
# this is finally where the action (alert) should be triggered from
testDBSUpload.algorithm()
return
def __call__(self, parameters):
"""
Handles the event with payload, by sending it to the threadpool.
"""
myThread = threading.currentThread()
#myThread.transaction.begin()
#payload = parameters #I think this is correct; it's all that I've got to go on
factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface")
createWorkArea = factory.loadObject("CreateWorkArea")
jobGroupID = parameters #There should only be one
#We should have, in the payload, exactly one jobGroup ID
createWorkArea.getNewJobGroup(jobGroupID)
createWorkArea.createJobGroupArea()
createWorkArea.createWorkArea()
#For testing purposes only
#DO NOT LEAVE ACTIVE!
#createWorkArea.cleanUpAll()
#myThread.transaction.commit()
return
def __init__(self, config):
RESTModel.__init__(self, config)
self.methods['POST'] = {
'plot': {
'version': 1,
'call': self.plot,
'args': ['type', 'data', 'url'],
'expires': 300
}
}
self.methods['GET'] = {
'plot': {
'version': 1,
'call': self.plot,
'args': ['type', 'data', 'url'],
'expires': 300
},
'doc': {
'version': 1,
'call': self.doc,
'args': ['type'],
'expires': 300
}
}
self.factory = WMFactory('plot_fairy',
'WMCore.HTTPFrontEnd.PlotFairy.Plots')
def __call__(self, parameters):
"""
Handles the event with payload, by sending it to the threadpool.
"""
myThread = threading.currentThread()
#myThread.transaction.begin()
#payload = parameters #I think this is correct; it's all that I've got to go on
factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface")
createWorkArea=factory.loadObject("CreateWorkArea")
jobGroupID = parameters #There should only be one
#We should have, in the payload, exactly one jobGroup ID
createWorkArea.getNewJobGroup(jobGroupID)
createWorkArea.createJobGroupArea()
createWorkArea.createWorkArea()
#For testing purposes only
#DO NOT LEAVE ACTIVE!
#createWorkArea.cleanUpAll()
#myThread.transaction.commit()
return
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject(classname="Feeder", getFromCache=False)
feeder(fileset)
except Exception as e:
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info(traceback.format_exc())
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def __init__(self ):
"""
The constructor creates separate instances
for objects that we do not want to share.
Objects that are used as a thread slave
Inherit from this to ensure they do not
conflict with these methods.
If needed the developer can add additional
objects as it sees fit in the threadslave
class that inherits from this.
"""
self.args = {}
# we also keep a reference to our component (we can
# use this for read only things in the argument list).
# assign this later
self.component = None
#a slave is created in its master thread so we can exploit
#this to get a reference to its dbfactory object.
myThread = threading.currentThread()
self.dbFactory = myThread.dbFactory
# get the procid from the mainthread msg service
# if we use this in testing it might not be there.
self.procid = 0
# we can potentially use mapping from messages to handlers
# to have one thread handling multiple message types.
self.messages = {}
# start loading some objects we need in this thread only
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject( classname = "Feeder", getFromCache = False )
feeder(fileset)
except Exception as e :
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info( traceback.format_exc() )
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
"WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"):
pluginFactory = WMFactory("specArgs", loc)
alteredClassName = "%sWorkloadFactory" % request_type
spec = pluginFactory.loadObject(classname=request_type, alteredClassName=alteredClassName)
specArgs = spec.getWorkloadArguments()
result = create_json_template_spec(specArgs)
return result
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"):
pluginFactory = WMFactory("specArgs", loc)
alteredClassName = "%sWorkloadFactory" % request_type
spec = pluginFactory.loadObject(classname = request_type, alteredClassName = alteredClassName)
specArgs = spec.getWorkloadArguments()
result = create_json_template_spec(specArgs)
return result
class Plotter(RESTModel):
'''
A class that generates a plot object to send to the formatter, possibly
downloading the data from a remote resource along the way
'''
def __init__(self, config):
RESTModel.__init__(self, config)
self.methods['POST'] = {'plot': {'version':1,
'call':self.plot,
'args': ['type', 'data', 'url'],
'expires': 300}}
self.methods['GET'] = {'plot': {'version':1,
'call':self.plot,
'args': ['type', 'data', 'url'],
'expires': 300},
'doc': {'version':1,
'call':self.doc,
'args': ['type'],
'expires':300}}
self.factory = WMFactory('plot_fairy',
'WMCore.HTTPFrontEnd.PlotFairy.Plots')
def plot(self, *args, **kwargs):
input = self.sanitise_input(args, kwargs, 'plot')
plot = self.factory.loadObject(input['type'])
return {'figure': plot(input['data'])}
def doc(self, *args, **kwargs):
input = self.sanitise_input(args, kwargs, 'doc')
plot = self.factory.loadObject(input['type'])
return {'doc': plot.doc()}
def validate_input(self, input, verb, method):
assert 'type' in input.keys(), "no type provided - what kind of plot do you want?"
if method=='doc':
return input
valid_data = {}
assert 'data' in input.keys() or 'url' in input.keys(), "neither data nor url provided - please provide at least one"
if 'url' in input.keys():
match = URL_REGEX.match(input['url'])
assert match != None, "`%s' is not a valid URL" % input['url']
host = match.group(1) + '://' + match.group(3)
if match.group(4):
host += match.group(4)
uri = match.group(5)
result,status,reason,fromcache = JSONRequests(host).get(uri)
valid_data.update(result)
if 'data' in input.keys():
valid_data.update(json.loads(urllib.unquote(input['data'])))
return {'data':valid_data,'type':input['type']}
def getDBinstance(config, namespace, name):
if config.backend.lower() == 'mysql':
backend = 'MySQL'
elif config.backend.lower() == 'oracle':
backend = 'Oracle'
#factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name = name, namespace = 'Databases.%s.%s.%s' % (namespace,backend,name))
return factory.loadObject( name )
def getDBinstance(config, namespace, name):
if config.backend.lower() == 'mysql':
backend = 'MySQL'
elif config.backend.lower() == 'oracle':
backend = 'Oracle'
#factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name = name, namespace = 'Databases.%s.%s.%s' % (namespace,backend,name))
return factory.loadObject( name )
def updateDSFileCount(self, dataset):
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database."+ \
myThread.dialect)
newDS = factory.loadObject("UpdateDSFileCount")
newDS.execute(dataset=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def updateDSFileCount(self, dataset):
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database."+ \
myThread.dialect)
newDS = factory.loadObject("UpdateDSFileCount")
newDS.execute(dataset=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def __init__(self, config, insertStates=False):
"""
__init__
BossAir should work with the standard config
structure of WMAgent
"""
WMConnectionBase.__init__(self, daoPackage="WMCore.BossAir")
myThread = threading.currentThread()
self.config = config
self.plugins = {}
self.states = []
self.jobs = []
self.pluginDir = config.BossAir.pluginDir
# This is the default state jobs are created in
self.newState = getattr(config.BossAir, 'newState', 'New')
# Get any proxy info
self.checkProxy = getattr(config.BossAir, 'checkProxy', False)
self.cert = getattr(config.BossAir, 'cert', None)
self.stateMachine = ChangeState(self.config)
# Create a factory to load plugins
self.pluginFactory = WMFactory("plugins", self.pluginDir)
self.daoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.deleteDAO = self.daoFactory(classname="DeleteJobs")
self.stateDAO = self.daoFactory(classname="NewState")
self.loadByWMBSDAO = self.daoFactory(classname="LoadByWMBSID")
self.updateDAO = self.daoFactory(classname="UpdateJobs")
self.newJobDAO = self.daoFactory(classname="NewJobs")
self.runningJobDAO = self.daoFactory(classname="LoadRunning")
self.completeJobDAO = self.daoFactory(classname="LoadComplete")
self.loadJobsDAO = self.daoFactory(classname="LoadByStatus")
self.completeDAO = self.daoFactory(classname="CompleteJob")
self.monitorDAO = self.daoFactory(classname="JobStatusForMonitoring")
self.states = None
self.loadPlugin(insertStates)
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
myThread = threading.currentThread()
factory = WMFactory("WMBS", "WMCore.WMBS")
destroy = factory.loadObject(myThread.dialect + ".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn=myThread.transaction.conn)
if not destroyworked:
raise Exception("Could not complete WMBS tear down.")
myThread.transaction.commit()
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
myThread = threading.currentThread()
factory = WMFactory("WMBS", "WMCore.WMBS")
destroy = factory.loadObject(myThread.dialect + ".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn=myThread.transaction.conn)
if not destroyworked:
raise Exception("Could not complete WMBS tear down.")
myThread.transaction.commit()
def __init__(self, config, dbsconfig = None):
"""
Initialise class members
"""
myThread = threading.currentThread()
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.bufferFactory = DAOFactory(package = "WMComponent.DBSBuffer.Database",
logger = myThread.logger,
dbinterface = myThread.dbi)
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
self.uploadToDBS = factory.loadObject("UploadToDBS")
addFactory = WMFactory("dbsBuffer",
"WMComponent.DBSBuffer.Database.Interface")
self.addToBuffer = addFactory.loadObject("AddToBuffer")
# Set DBSInterface
self.dbsInterface = DBSInterface(config = config)
# Set DAOs
self.setBlock = self.bufferFactory(classname = "DBSBufferFiles.SetBlock")
self.setStatus = self.bufferFactory(classname = "DBSBufferFiles.SetStatus")
# Set config parameters
self.maxBlockFiles = self.config.DBSInterface.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSInterface.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSInterface.DBSBlockMaxSize
self.doMigration = getattr(self.config.DBSInterface, 'doGlobalMigration', True)
logging.debug("Initializing with maxFiles %i, maxTime %i, maxSize %i" % (self.maxBlockFiles,
self.maxBlockTime,
self.maxBlockSize))
if dbsconfig == None:
self.dbsconfig = config
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "DBSUpload")
return
def __init__(self, config):
"""
Initialise class members
"""
#Need a better way to test this without turning off this next line
BaseDaemon.__init__(self, config, 'AsyncTransfer')
self.dropbox_dir = '%s/dropbox/outputs' % self.config.componentDir
if not os.path.isdir(self.dropbox_dir):
try:
os.makedirs(self.dropbox_dir)
except OSError as e:
if e.errno == errno.EEXIST:
pass
else:
self.logger.error('Unknown error in mkdir' % e.errno)
raise
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.logger.debug('Connected to CouchDB')
self.pool = Pool(processes=self.config.pool_size)
try:
self.phedex = PhEDEx(responseType='xml', dict = {'key': self.config.opsProxy, 'cert': self.config.opsProxy})
except Exception as e:
self.logger.exception('PhEDEx exception: %s' % e)
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.schedAlgoDir, namespace = self.config.schedAlgoDir)
result_list = []
current_running = []
def preInitialization(self):
"""
__preInitialization__
"""
if not hasattr(self.config.PilotManagerComponent, "pilotManagerHandler"):
self.config.PilotManagerComponent.pilotManagerHandler = 'PilotManager.Handler.PilotManagerHandler'
factory = WMFactory('PilotManager');
self.messages["NewPilotJob"] = factory.loadObject(\
self.config.PilotManagerComponent.pilotManagerHandler, self)
self.messages["SubmitPilotJob"] = factory.loadObject(\
self.config.PilotManagerComponent.pilotManagerHandler, self)
#create pilot tar file
self.createPilotTar()
class GeneratorFactory(WMFactory):
"""
_GeneratorFactory_
Instantiate a WMFactory instance with the appropriate namespace
"""
def __init__(self):
self.factory = WMFactory(self.__class__.__name__,
"WMCore.JobSplitting.Generators")
def getGenerator(self, generatorName, **options):
"""
_getGenerator_
factory method to return a step template instance based on the
name of the step
"""
args = {}
args.update(options)
try:
return self.factory.loadObject(generatorName, args)
except WMException, wmEx:
msg = "GeneratorFactory Unable to load Object: %s" % generatorName
raise GeneratorFactoryException(msg)
except Exception, ex:
msg = "Error creating object %s in GeneratorFactory:\n" % generatorName
msg += str(ex)
raise GeneratorFactoryException(msg)
def setBlockStatus(self, block, locations, openStatus = 0, time = 0):
"""
_setBlockStatus_
Adds information about the block from DBSWriter to the database
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
newDS = factory.loadObject("SetBlockStatus")
newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \
conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def _makeIndex(self):
"""
Create an index page, either from the configured page or a generic default
welcome page.
"""
globalconf = self.appconfig.dictionary_()
if hasattr(self.appconfig, 'index'):
factory = WMFactory('webtools_factory')
view = getattr(self.appconfig.views.active, globalconf['index'])
del globalconf['views']
del globalconf['index']
self._mountPage(view, globalconf, factory, is_index=True)
else:
cherrypy.log.error_log.info("No index defined for %s - instantiating default Welcome page"
% (self.app))
namesAndDocstrings = []
# make a default Welcome
for view in self.appconfig.views.active:
if not getattr(view, "hidden", False):
viewName = view._internal_name
if 'instances' in globalconf.keys():
for instance in globalconf['instances']:
mount_point = '/%s/%s/%s' % (self.app.lower(), instance, viewName)
viewObj = cherrypy.tree.apps[mount_point].root
docstring = viewObj.__doc__
namesAndDocstrings.append(('%s/%s' % (instance, viewName), docstring))
else:
mount_point = '/%s/%s' % (self.app.lower(), viewName)
viewObj = cherrypy.tree.apps[mount_point].root
docstring = viewObj.__doc__
namesAndDocstrings.append((viewName, docstring))
cherrypy.tree.mount(Welcome(namesAndDocstrings), "/%s" % self.app.lower())
def setBlockStatus(self, block, locations, openStatus=0, time=0):
"""
_setBlockStatus_
Adds information about the block from DBSWriter to the database
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
newDS = factory.loadObject("SetBlockStatus")
newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \
conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
pluginPath = getattr(self.config.RetryManager, "pluginPath",
"WMComponent.RetryManager.PlugIns")
self.pluginFactory = WMFactory("plugins", pluginPath)
self.changeState = ChangeState(self.config)
self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")
# initialize the alert framework (if available) (self.sendAlert())
self.initAlerts(compName = "RetryManager")
try:
pluginName = getattr(self.config.RetryManager, 'pluginName', 'DefaultRetryAlgo')
self.plugin = self.pluginFactory.loadObject(classname = pluginName,
args = config)
except Exception, ex:
msg = "Error loading plugin %s on path %s\n" % (pluginName, pluginPath)
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
raise RetryManagerException(msg)
def __init__(self, config):
"""
Initialise class members
"""
BaseDaemon.__init__(self, config, 'RetryManager')
if self.config.isOracle:
self.oracleDB = HTTPRequests(self.config.oracleDB,
self.config.opsProxy,
self.config.opsProxy)
else:
try:
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
except Exception as e:
self.logger.exception('A problem occured when connecting to couchDB: %s' % e)
raise
self.logger.debug('Connected to files DB')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.retryAlgoDir, namespace=self.config.retryAlgoDir)
try:
self.plugin = self.factory.loadObject(self.config.algoName, self.config,
getFromCache=False, listFlag=True)
except Exception as ex:
msg = "Error loading plugin %s on path %s\n" % (self.config.algoName,
self.config.retryAlgoDir)
msg += str(ex)
self.logger.error(msg)
raise RetryManagerException(msg)
self.cooloffTime = self.config.cooloffTime
def _loadPages(self):
"""
Load up all the pages in the configuration
"""
factory = WMFactory('webtools_factory')
globalconf = self.appconfig.dictionary_()
del globalconf['views']
the_index = ''
if 'index' in globalconf:
the_index = globalconf['index']
del globalconf['index']
for view in self.appconfig.views.active:
# Iterate through each view's configuration and instantiate the class
if view._internal_name != the_index:
if 'instances' in globalconf:
for instance in globalconf['instances']:
self._mountPage(view, globalconf, factory, instance)
else:
self._mountPage(view, globalconf, factory)
if hasattr(self.appconfig.views, 'maintenance'):
# for i in self.appconfig.views.maintenance:
# TODO: Show a maintenance page with a 503 Service Unavailable header
pass
class GeneratorFactory(WMFactory):
"""
_GeneratorFactory_
Instantiate a WMFactory instance with the appropriate namespace
"""
def __init__(self):
self.factory = WMFactory(self.__class__.__name__,
"WMCore.JobSplitting.Generators")
def getGenerator(self, generatorName, **options):
"""
_getGenerator_
factory method to return a step template instance based on the
name of the step
"""
args = {}
args.update(options)
try:
return self.factory.loadObject(generatorName,
args)
except WMException, wmEx:
msg = "GeneratorFactory Unable to load Object: %s" % generatorName
raise GeneratorFactoryException(msg)
except Exception, ex:
msg = "Error creating object %s in GeneratorFactory:\n" % generatorName
msg += str(ex)
raise GeneratorFactoryException(msg)
def preInitialization(self):
if not hasattr(self.config.PilotMonitorComponent, "pilotMonitorHandler"):
self.config.PilotMonitorComponent.pilotMonitorHandler = \
'PilotMonitor.Handler.PilotMonitorHandler'
factory = WMFactory("PilotMonitor");
self.messages["MonitorPilots"] = factory.loadObject(\
self.config.PilotMonitorComponent.pilotMonitorHandler, self)
tqconfig = Configuration.loadConfigurationFile( \
self.config.PilotMonitorComponent.TQConfig )
myThread = threading.currentThread()
self.logger = myThread.logger
self.logger.debug( tqconfig)
self.tqStateApi = TQStateApi(self.logger, tqconfig, None)
def clearDatabase(self, modules = []):
"""
Enables clearing particular tables in the database
Associated to modules. Note this only works if there
is the module has a Destroy class. Beware that this
might not work if there are table dependencies.
"""
myThread = threading.currentThread()
for module in modules:
factory = WMFactory("clear", module)
destroy = factory.loadObject(myThread.dialect+".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn = myThread.transaction.conn)
if not destroyworked:
raise Exception(module +" tables could not be destroyed")
myThread.transaction.commit()
del factory
def __init__(self, logPath=None, config=None):
threading.Thread.__init__(self)
self.doMonitoring = True
self._Finished = threading.Event()
self._EndOfJob = threading.Event()
self._NewTask = threading.Event()
self._JobKilled = threading.Event()
self._RunUpdate = threading.Event()
self._Interval = 120.0
self._Monitors = []
# Right now we join this, because we don't know
# Where we'll be when we need this.
self.logPath = os.path.join(os.getcwd(), logPath)
self.factory = WMFactory(self.__class__.__name__,
"WMCore.WMRuntime.Monitors")
def initInThread(self):
"""
Called during thread initialization. Loads the
backend for this instance.
"""
# make sure you instantiate the super class method.
ThreadSlave.initInThread(self)
# load queries for backend.
myThread = threading.currentThread()
factory = WMFactory("default", \
"TQComp.Database."+myThread.dialect)
# make sure you do not overload attributes
# defined in your parent class
self.queries = factory.loadObject("Queries")
logging.debug("ParentSlave initialized")
def testE_NoMigration(self):
"""
_NoMigration_
Test the DBSUpload system with no global migration
"""
myThread = threading.currentThread()
config = self.createConfig()
self.injectWorkflow(MaxWaitTime=3)
config.DBSInterface.doGlobalMigration = False
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config=config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('Open', )])
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1"
)[0].fetchall()
for r in result:
self.assertEqual(r[0], 'GLOBAL')
return
def findAlgos(self, dataset):
"""
Call to findAlgos
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
findAlgos = factory.loadObject("FindAlgos")
# Add the file to the buffer (API Call)
#results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction())
results = findAlgos.execute(datasetInfo=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return results
def testE_NoMigration(self):
"""
_NoMigration_
Test the DBSUpload system with no global migration
"""
myThread = threading.currentThread()
config = self.createConfig()
config.DBSInterface.DBSBlockMaxTime = 3
config.DBSInterface.doGlobalMigration = False
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config = config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef = True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config = config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS',), ('Open',)])
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1")[0].fetchall()
for r in result:
self.assertEqual(r[0], 'GLOBAL')
return
class LFNSourceDuplicator(BaseWorkerThread):
"""
_LFNSourceDuplicator_
Load plugin to get the result of the source database query.
Duplicates the result got into the local database.
"""
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.AsyncTransfer
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
# Asynch db
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
return
def algorithm(self, parameters = None):
"""
_algorithm_
Load the plugin of a db source which load its view.
Duplicates the results got from the plugin in Async database.
"""
self.logger.debug('Duplication algorithm begins')
try:
duplicator = self.factory.loadObject(self.config.pluginName, args = [self.config, self.logger], getFromCache = True, listFlag = True)
except ImportError,e :
msg = "plugin \'%s\' unknown" % self.config.pluginName
self.logger.info(msg)
self.logger.info(e)
for doc in duplicator():
self.db.queue(doc, True, ['AsyncTransfer/ftscp'] )
self.logger.debug("doc queued %s" %doc)
self.db.commit(viewlist=['AsyncTransfer/ftscp'])
self.logger.debug('Duplication done')
return
def runDas(self, func, data, expires):
"""
Run a query and produce a dictionary for DAS formatting
"""
start_time = time.time()
results = func(self, data)
call_time = time.time() - start_time
res_expire = make_timestamp(expires)
if isinstance(results, list):
if len(results) > 0:
row = results[0]
else:
row = None
else:
row = results
if isinstance(row, str):
row = '"%s"' % row
try:
factory = WMFactory('webtools_factory')
obj = factory.loadObject(self.config.model.object, self.config)
res_version = obj.version
except Exception:
res_version = 'unknown'
keyhash = hashlib.md5()
if not isinstance(results, (str, bytes)):
keyhash.update(encodeUnicodeToBytesConditional(str(results), condition=PY3))
else:
keyhash.update(encodeUnicodeToBytesConditional(results, condition=PY3))
res_checksum = keyhash.hexdigest()
dasdata = {'application': '%s.%s' % (self.config.application, func.__name__),
'request_timestamp': start_time,
'request_url': request.base + request.path_info + '?' + request.query_string,
'request_method': request.method,
'request_params': request.params,
'response_version': res_version,
'response_expires': res_expire,
'response_checksum': res_checksum,
'request_call': func.__name__,
'call_time': call_time,
'results': results,
}
return dasdata
def __init__(self, config):
self.config = config # Saves the config because Page needs it
self.encoder = json.JSONEncoder()
# The full URL of this openid server
if hasattr(config, 'server_url'):
self.base_path = '%s/' % config.server_url
else:
self.base_path = '%s/' % cherrypy.url()
# Now instantiates the OpenID Store (used for nonces and associations)
if config.store == 'filestore':
store = FileOpenIDStore(config.store_path)
elif config.store == 'dbstore':
store = DBOidStore(config.store_source)
else:
store = MemoryStore()
# Now instantiates the OpenID server
self.openid = server.Server(store, self.base_path + 'openidserver')
# Record how to get user details out...
factory = WMFactory('oid_factory')
self.userstore = factory.loadObject(config.users_store.object,
config.users_store)
# List of remote sites allowed to request auth/authz to this server
self.truststore = factory.loadObject(config.trust_store.object,
config.trust_store)
# This is a thread safe object to display web pages/forms related to
# the oid server. I put it on a separate class to avoid blowing
# the main class with html code.
self.webpages = OidServerWebPages()
# *** Should enable cherrypy session somewhere... sorry
cherrypy.config.update({
'tools.sessions.on': True,
'tools.sessions.storage_type': 'ram',
#'tools.sessions.storage_type': "file",
#'tools.sessions.storage_path': "/tmp/oidserver-sessions",
'tools.sessions.timeout': 60,
'tools.sessions.name': 'oidserver_sid'
})
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
pluginPath = getattr(self.config.RetryManager, "pluginPath",
"WMComponent.RetryManager.PlugIns")
self.pluginFactory = WMFactory("plugins", pluginPath)
self.changeState = ChangeState(self.config)
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
# initialize the alert framework (if available) (self.sendAlert())
self.initAlerts(compName="RetryManager")
# get needed plugins
self.plugins = {}
self.typePluginsAssoc = getattr(self.config.RetryManager, 'plugins',
{})
self.typePluginsAssoc.setdefault('default', 'DefaultRetryAlgo')
for pluginName in self.typePluginsAssoc.values():
try:
plugin = self.pluginFactory.loadObject(classname=pluginName,
args=config)
self.plugins[pluginName] = plugin
except Exception as ex:
msg = "Error loading plugin %s on path %s\n" % (pluginName,
pluginPath)
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
raise RetryManagerException(msg)
return
def testD(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
self.tempDir = self.testInit.generateWorkDir(config)
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent2")
os.makedirs(config.TestComponent.componentDir)
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent=True)
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir,
"Daemon.xml")
details = Details(daemonFile)
print('Is component alive: ' + str(details.isAlive()))
# create msgService to send stop message.
myThread = threading.currentThread()
factory = WMFactory("msgService", "WMCore.MsgService." + \
myThread.dialect)
myThread.transaction = Transaction(myThread.dbi)
msgService = factory.loadObject("MsgService")
msgService.registerAs("HarnessTest")
myThread.transaction.commit()
print(
'Publish a stop message to test if the component shutsdown gracefully'
)
myThread.transaction.begin()
msg = {'name': 'Stop', 'payload': ''}
msgService.publish(msg)
myThread.transaction.commit()
msgService.finish()
while details.isAlive():
print('Component has not received stop message')
time.sleep(2)
print('Daemon shutdown gracefully')
def findAlgos(self, dataset):
"""
Call to findAlgos
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
findAlgos = factory.loadObject("FindAlgos")
# Add the file to the buffer (API Call)
#results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction())
results = findAlgos.execute(datasetInfo=dataset,
conn=self.getDBConn(),
transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return results
def runDas(self, func, data, expires):
"""
Run a query and produce a dictionary for DAS formatting
"""
start_time = time.time()
results = func(self, data)
call_time = time.time() - start_time
res_expire = make_timestamp(expires)
if type(results) is types.ListType:
if len(results) > 0:
row = results[0]
else:
row = None
else:
row = results
if type(row) is types.StringType:
row = '"%s"' % row
try:
factory = WMFactory('webtools_factory')
object = factory.loadObject(self.config.model.object, self.config)
res_version = object.version
except:
res_version = 'unknown'
keyhash = hashlib.md5()
keyhash.update(str(results))
res_checksum = keyhash.hexdigest()
dasdata = {'application':'%s.%s' % (self.config.application, func.__name__),
'request_timestamp': start_time,
'request_url': request.base + request.path_info + '?' + \
request.query_string,
'request_method' : request.method,
'request_params' : request.params,
'response_version': res_version,
'response_expires': res_expire,
'response_checksum': res_checksum,
'request_call': func.__name__,
'call_time': call_time,
'results': results,
}
return dasdata
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def preInitialization(self):
"""
Add required worker modules to work threads
"""
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['AddWorkflowToManage'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManage", self)
self.messages['RemoveWorkflowFromManagement'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagement", self)
self.messages['AddToWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationList", self)
self.messages['RemoveFromWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationList", self)
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkflowManager.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(WorkflowManagerPoller(), \
pollInterval)
def __init__(self, config):
myThread = threading.currentThread()
self.config = config
self.dbsurl = self.config.DBSUpload.dbsurl
self.dbsversion = self.config.DBSUpload.dbsversion
self.uploadFileMax = self.config.DBSUpload.uploadFileMax
self.DBSMaxFiles = self.config.DBSUpload.DBSMaxFiles
self.DBSMaxSize = self.config.DBSUpload.DBSMaxSize
self.DBSBlockTimeout = self.config.DBSUpload.DBSBlockTimeout
self.dbswriter = DBSWriter(self.dbsurl, level='ERROR', user='******', version=self.dbsversion, \
globalDBSUrl = self.config.DBSUpload.globalDBSUrl, \
globalVersion = self.config.DBSUpload.globalDBSVer)
self.daoFactory = DAOFactory(package="WMComponent.DBSBuffer.Database",
logger=logging,
dbinterface=myThread.dbi)
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
self.dbinterface = factory.loadObject("UploadToDBS")