def __init__(self, config, dbsconfig=None):
"""
Initialise class members
"""
myThread = threading.currentThread()
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.bufferFactory = DAOFactory(
package="WMComponent.DBSBuffer.Database",
logger=myThread.logger,
dbinterface=myThread.dbi)
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
self.uploadToDBS = factory.loadObject("UploadToDBS")
addFactory = WMFactory("dbsBuffer",
"WMComponent.DBSBuffer.Database.Interface")
self.addToBuffer = addFactory.loadObject("AddToBuffer")
# Set DBSInterface
self.dbsInterface = DBSInterface(config=config)
# Set DAOs
self.setBlock = self.bufferFactory(classname="DBSBufferFiles.SetBlock")
self.setStatus = self.bufferFactory(
classname="DBSBufferFiles.SetStatus")
# Set config parameters
self.maxBlockFiles = self.config.DBSInterface.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSInterface.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSInterface.DBSBlockMaxSize
self.doMigration = getattr(self.config.DBSInterface,
'doGlobalMigration', True)
logging.debug(
"Initializing with maxFiles %i, maxTime %i, maxSize %i" %
(self.maxBlockFiles, self.maxBlockTime, self.maxBlockSize))
if dbsconfig == None:
self.dbsconfig = config
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName="DBSUpload")
return
def _loadPages(self):
"""
Load up all the pages in the configuration
"""
factory = WMFactory('webtools_factory')
globalconf = self.appconfig.dictionary_()
del globalconf['views']
the_index = ''
if 'index' in globalconf:
the_index = globalconf['index']
del globalconf['index']
for view in self.appconfig.views.active:
# Iterate through each view's configuration and instantiate the class
if view._internal_name != the_index:
if 'instances' in globalconf:
for instance in globalconf['instances']:
self._mountPage(view, globalconf, factory, instance)
else:
self._mountPage(view, globalconf, factory)
if hasattr(self.appconfig.views, 'maintenance'):
# for i in self.appconfig.views.maintenance:
# TODO: Show a maintenance page with a 503 Service Unavailable header
pass
def pollExternal(self, fileset):
"""
Call relevant external source and get file details
"""
logging.debug("Feeder name %s" % (fileset.name).split(":")[1])
try:
factory = WMFactory("default", \
"WMCore.WMBSFeeder." + (fileset.name).split(":")[1])
feeder = factory.loadObject(classname="Feeder", getFromCache=False)
feeder(fileset)
except Exception as e:
msg = "Feeder plugin \'%s\' unknown" \
% (fileset.name).split(":")[1]
logging.info(msg)
logging.info(e)
logging.info(traceback.format_exc())
logging.info("aborting poll for...closing fileset")
fileset.markOpen(False)
fileset.commit()
return fileset
def preInitialization(self):
"""
Add required worker modules to work threads
"""
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['AddWorkflowToManage'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddWorkflowToManage", self)
self.messages['RemoveWorkflowFromManagement'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveWorkflowFromManagement", self)
self.messages['AddToWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.AddToWorkflowManagementLocationList", self)
self.messages['RemoveFromWorkflowManagementLocationList'] = \
factory.loadObject(\
"WMComponent.WorkflowManager.Handler.RemoveFromWorkflowManagementLocationList", self)
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.WorkflowManager.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(WorkflowManagerPoller(), \
pollInterval)
def __init__(self):
"""
The constructor creates separate instances
for objects that we do not want to share.
Objects that are used as a thread slave
Inherit from this to ensure they do not
conflict with these methods.
If needed the developer can add additional
objects as it sees fit in the threadslave
class that inherits from this.
"""
self.args = {}
# we also keep a reference to our component (we can
# use this for read only things in the argument list).
# assign this later
self.component = None
#a slave is created in its master thread so we can exploit
#this to get a reference to its dbfactory object.
myThread = threading.currentThread()
self.dbFactory = myThread.dbFactory
# get the procid from the mainthread msg service
# if we use this in testing it might not be there.
self.procid = 0
# we can potentially use mapping from messages to handlers
# to have one thread handling multiple message types.
self.messages = {}
# start loading some objects we need in this thread only
factory = WMFactory("threadPool", "WMCore.ThreadPool."+ \
myThread.dialect)
self.query = factory.loadObject("Queries")
def preInitialization(self):
print "JobMaker.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
self.messages['MakeJob'] = \
factory.loadObject('WMCore.WMSpec.Makers.Handlers.MakeJob', self)
def _makeIndex(self):
"""
Create an index page, either from the configured page or a generic default
welcome page.
"""
globalconf = self.appconfig.dictionary_()
if hasattr(self.appconfig, 'index'):
factory = WMFactory('webtools_factory')
view = getattr(self.appconfig.views.active, globalconf['index'])
del globalconf['views']
del globalconf['index']
self._mountPage(view, globalconf, factory, is_index=True)
else:
cherrypy.log.error_log.info("No index defined for %s - instantiating default Welcome page"
% (self.app))
namesAndDocstrings = []
# make a default Welcome
for view in self.appconfig.views.active:
if not getattr(view, "hidden", False):
viewName = view._internal_name
if 'instances' in globalconf.keys():
for instance in globalconf['instances']:
mount_point = '/%s/%s/%s' % (self.app.lower(), instance, viewName)
viewObj = cherrypy.tree.apps[mount_point].root
docstring = viewObj.__doc__
namesAndDocstrings.append(('%s/%s' % (instance, viewName), docstring))
else:
mount_point = '/%s/%s' % (self.app.lower(), viewName)
viewObj = cherrypy.tree.apps[mount_point].root
docstring = viewObj.__doc__
namesAndDocstrings.append((viewName, docstring))
cherrypy.tree.mount(Welcome(namesAndDocstrings), "/%s" % self.app.lower())
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.AsyncTransfer
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
# Asynch db
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
return
def __init__(self, config):
RESTModel.__init__(self, config)
self.methods['POST'] = {
'plot': {
'version': 1,
'call': self.plot,
'args': ['type', 'data', 'url'],
'expires': 300
}
}
self.methods['GET'] = {
'plot': {
'version': 1,
'call': self.plot,
'args': ['type', 'data', 'url'],
'expires': 300
},
'doc': {
'version': 1,
'call': self.doc,
'args': ['type'],
'expires': 300
}
}
self.factory = WMFactory('plot_fairy',
'WMCore.HTTPFrontEnd.PlotFairy.Plots')
def testF_DBSUploadQueueSizeCheckForAlerts(self):
"""
Test will not trigger a real alert being sent unless doing some
mocking of the methods used during DBSUploadPoller.algorithm() ->
DBSUploadPoller.uploadBlocks() method.
As done here, it probably can't be deterministic, yet the feature
shall be checked.
"""
sizeLevelToTest = 1
myThread = threading.currentThread()
config = self.createConfig()
# threshold / value to check
config.DBSUpload.alertUploadQueueSize = sizeLevelToTest
# without this uploadBlocks method returns immediately
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = sizeLevelToTest + 1
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# load components that are necessary to check status
# (this seems necessary, else some previous tests started failing)
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config = config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef = True)
testDBSUpload = DBSUploadPoller(config)
# this is finally where the action (alert) should be triggered from
testDBSUpload.algorithm()
return
def setSchema(self, modules=None, params=None):
"""
Creates the schema in the database based on the modules
input.
This method needs to have been preceded by the
setDatabaseConnection.
"""
modules = modules or []
myThread = threading.currentThread()
parameters = None
flag = False
# Set up for typical DBCreator format: logger, dbi, params
if params != None:
parameters = [None, None, params]
flag = True
myThread.transaction.begin()
for factoryName in modules:
# need to create these tables for testing.
# notice the default structure: <dialect>/Create
factory = WMFactory(factoryName, factoryName + "." + myThread.dialect)
create = factory.loadObject("Create", args=parameters, listFlag=flag)
createworked = create.execute(conn=myThread.transaction.conn,
transaction=myThread.transaction)
if createworked:
logging.debug("Tables for " + factoryName + " created")
else:
logging.debug("Tables " + factoryName + " could not be created.")
myThread.transaction.commit()
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
"WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def __call__(self, parameters):
"""
Handles the event with payload, by sending it to the threadpool.
"""
myThread = threading.currentThread()
#myThread.transaction.begin()
#payload = parameters #I think this is correct; it's all that I've got to go on
factory = WMFactory("JobMaker", "WMCore.WMSpec.Makers.Interface")
createWorkArea = factory.loadObject("CreateWorkArea")
jobGroupID = parameters #There should only be one
#We should have, in the payload, exactly one jobGroup ID
createWorkArea.getNewJobGroup(jobGroupID)
createWorkArea.createJobGroupArea()
createWorkArea.createWorkArea()
#For testing purposes only
#DO NOT LEAVE ACTIVE!
#createWorkArea.cleanUpAll()
#myThread.transaction.commit()
return
def get_request_template_from_type(request_type, loc="WMSpec.StdSpecs"):
pluginFactory = WMFactory("specArgs", loc)
alteredClassName = "%sWorkloadFactory" % request_type
spec = pluginFactory.loadObject(classname=request_type, alteredClassName=alteredClassName)
specArgs = spec.getWorkloadArguments()
result = create_json_template_spec(specArgs)
return result
def getDBinstance(config, namespace, name):
if config.backend.lower() == 'mysql':
backend = 'MySQL'
elif config.backend.lower() == 'oracle':
backend = 'Oracle'
#factory = WMFactory(name = 'TaskQuery', namespace = 'Databases.TaskDB.%s.Task' % backend)
factory = WMFactory(name = name, namespace = 'Databases.%s.%s.%s' % (namespace,backend,name))
return factory.loadObject( name )
def updateDSFileCount(self, dataset):
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsBuffer", "WMComponent.DBSBuffer.Database."+ \
myThread.dialect)
newDS = factory.loadObject("UpdateDSFileCount")
newDS.execute(dataset=dataset, conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def __init__(self, config, insertStates=False):
"""
__init__
BossAir should work with the standard config
structure of WMAgent
"""
WMConnectionBase.__init__(self, daoPackage="WMCore.BossAir")
myThread = threading.currentThread()
self.config = config
self.plugins = {}
self.states = []
self.jobs = []
self.pluginDir = config.BossAir.pluginDir
# This is the default state jobs are created in
self.newState = getattr(config.BossAir, 'newState', 'New')
# Get any proxy info
self.checkProxy = getattr(config.BossAir, 'checkProxy', False)
self.cert = getattr(config.BossAir, 'cert', None)
self.stateMachine = ChangeState(self.config)
# Create a factory to load plugins
self.pluginFactory = WMFactory("plugins", self.pluginDir)
self.daoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.deleteDAO = self.daoFactory(classname="DeleteJobs")
self.stateDAO = self.daoFactory(classname="NewState")
self.loadByWMBSDAO = self.daoFactory(classname="LoadByWMBSID")
self.updateDAO = self.daoFactory(classname="UpdateJobs")
self.newJobDAO = self.daoFactory(classname="NewJobs")
self.runningJobDAO = self.daoFactory(classname="LoadRunning")
self.completeJobDAO = self.daoFactory(classname="LoadComplete")
self.loadJobsDAO = self.daoFactory(classname="LoadByStatus")
self.completeDAO = self.daoFactory(classname="CompleteJob")
self.monitorDAO = self.daoFactory(classname="JobStatusForMonitoring")
self.states = None
self.loadPlugin(insertStates)
return
def tearDown(self):
"""
_tearDown_
Drop all the WMBS tables.
"""
myThread = threading.currentThread()
factory = WMFactory("WMBS", "WMCore.WMBS")
destroy = factory.loadObject(myThread.dialect + ".Destroy")
myThread.transaction.begin()
destroyworked = destroy.execute(conn=myThread.transaction.conn)
if not destroyworked:
raise Exception("Could not complete WMBS tear down.")
myThread.transaction.commit()
def preInitialization(self):
print "DBSUpload.preInitialization"
# use a factory to dynamically load handlers.
factory = WMFactory('generic')
# Add event loop to worker manager
myThread = threading.currentThread()
pollInterval = self.config.DBSUpload.pollInterval
logging.info("Setting poll interval to %s seconds" % pollInterval)
myThread.workerThreadManager.addWorker(DBSUploadPoller(self.config), pollInterval)
return
def setBlockStatus(self, block, locations, openStatus=0, time=0):
"""
_setBlockStatus_
Adds information about the block from DBSWriter to the database
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
newDS = factory.loadObject("SetBlockStatus")
newDS.execute(block = block, locations = locations, open_status = openStatus, time = time, \
conn = self.getDBConn(), transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return
def __init__(self, config):
"""
Initialise class members
"""
#Need a better way to test this without turning off this next line
BaseDaemon.__init__(self, config, 'DBSPublisher')
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.schedAlgoDir,
namespace=self.config.schedAlgoDir)
self.pool = Pool(processes=self.config.publication_pool_size)
def __init__(self, logPath=None, config=None):
threading.Thread.__init__(self)
self.doMonitoring = True
self._Finished = threading.Event()
self._EndOfJob = threading.Event()
self._NewTask = threading.Event()
self._JobKilled = threading.Event()
self._RunUpdate = threading.Event()
self._Interval = 120.0
self._Monitors = []
# Right now we join this, because we don't know
# Where we'll be when we need this.
self.logPath = os.path.join(os.getcwd(), logPath)
self.factory = WMFactory(self.__class__.__name__,
"WMCore.WMRuntime.Monitors")
def testE_NoMigration(self):
"""
_NoMigration_
Test the DBSUpload system with no global migration
"""
myThread = threading.currentThread()
config = self.createConfig()
self.injectWorkflow(MaxWaitTime=3)
config.DBSInterface.doGlobalMigration = False
config.DBSUpload.pollInterval = 4
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
# Load components that are necessary to check status
factory = WMFactory("dbsUpload",
"WMComponent.DBSUpload.Database.Interface")
dbinterface = factory.loadObject("UploadToDBS")
dbsInterface = DBSInterface(config=config)
localAPI = dbsInterface.getAPIRef()
globeAPI = dbsInterface.getAPIRef(globalRef=True)
# In the first round we should create blocks for the first dataset
# The child dataset should not be handled until the parent is uploaded
testDBSUpload = DBSUploadPoller(config=config)
testDBSUpload.algorithm()
# First, see if there are any blocks
# One in DBS, one not in DBS
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 2)
self.assertEqual(result, [('InGlobalDBS', ), ('Open', )])
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_file WHERE dataset_algo = 1"
)[0].fetchall()
for r in result:
self.assertEqual(r[0], 'GLOBAL')
return
class RetryManagerDaemon(BaseWorkerThread):
"""
_RetryManagerPoller_
Polls for Files in CoolOff State and attempts to retry them
based on the requirements in the selected plugin
"""
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config.RetryManager
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
try:
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
except Exception, e:
self.logger.exception(
'A problem occured when connecting to couchDB: %s' % e)
raise
self.logger.debug('Connected to files DB')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.retryAlgoDir,
namespace=self.config.retryAlgoDir)
try:
self.plugin = self.factory.loadObject(self.config.algoName,
self.config,
getFromCache=False,
listFlag=True)
except Exception, ex:
msg = "Error loading plugin %s on path %s\n" % (
self.config.algoName, self.config.retryAlgoDir)
msg += str(ex)
self.logger.error(msg)
raise RetryManagerException(msg)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
pluginPath = getattr(self.config.RetryManager, "pluginPath",
"WMComponent.RetryManager.PlugIns")
self.pluginFactory = WMFactory("plugins", pluginPath)
self.changeState = ChangeState(self.config)
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
# initialize the alert framework (if available) (self.sendAlert())
self.initAlerts(compName="RetryManager")
# get needed plugins
self.plugins = {}
self.typePluginsAssoc = getattr(self.config.RetryManager, 'plugins',
{})
self.typePluginsAssoc.setdefault('default', 'DefaultRetryAlgo')
for pluginName in self.typePluginsAssoc.values():
try:
plugin = self.pluginFactory.loadObject(classname=pluginName,
args=config)
self.plugins[pluginName] = plugin
except Exception as ex:
msg = "Error loading plugin %s on path %s\n" % (pluginName,
pluginPath)
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
raise RetryManagerException(msg)
return
def runDas(self, func, data, expires):
"""
Run a query and produce a dictionary for DAS formatting
"""
start_time = time.time()
results = func(self, data)
call_time = time.time() - start_time
res_expire = make_timestamp(expires)
if isinstance(results, list):
if len(results) > 0:
row = results[0]
else:
row = None
else:
row = results
if isinstance(row, str):
row = '"%s"' % row
try:
factory = WMFactory('webtools_factory')
obj = factory.loadObject(self.config.model.object, self.config)
res_version = obj.version
except Exception:
res_version = 'unknown'
keyhash = hashlib.md5()
if not isinstance(results, (str, bytes)):
keyhash.update(encodeUnicodeToBytesConditional(str(results), condition=PY3))
else:
keyhash.update(encodeUnicodeToBytesConditional(results, condition=PY3))
res_checksum = keyhash.hexdigest()
dasdata = {'application': '%s.%s' % (self.config.application, func.__name__),
'request_timestamp': start_time,
'request_url': request.base + request.path_info + '?' + request.query_string,
'request_method': request.method,
'request_params': request.params,
'response_version': res_version,
'response_expires': res_expire,
'response_checksum': res_checksum,
'request_call': func.__name__,
'call_time': call_time,
'results': results,
}
return dasdata
def __init__(self, config):
self.config = config # Saves the config because Page needs it
self.encoder = json.JSONEncoder()
# The full URL of this openid server
if hasattr(config, 'server_url'):
self.base_path = '%s/' % config.server_url
else:
self.base_path = '%s/' % cherrypy.url()
# Now instantiates the OpenID Store (used for nonces and associations)
if config.store == 'filestore':
store = FileOpenIDStore(config.store_path)
elif config.store == 'dbstore':
store = DBOidStore(config.store_source)
else:
store = MemoryStore()
# Now instantiates the OpenID server
self.openid = server.Server(store, self.base_path + 'openidserver')
# Record how to get user details out...
factory = WMFactory('oid_factory')
self.userstore = factory.loadObject(config.users_store.object,
config.users_store)
# List of remote sites allowed to request auth/authz to this server
self.truststore = factory.loadObject(config.trust_store.object,
config.trust_store)
# This is a thread safe object to display web pages/forms related to
# the oid server. I put it on a separate class to avoid blowing
# the main class with html code.
self.webpages = OidServerWebPages()
# *** Should enable cherrypy session somewhere... sorry
cherrypy.config.update({
'tools.sessions.on': True,
'tools.sessions.storage_type': 'ram',
#'tools.sessions.storage_type': "file",
#'tools.sessions.storage_path': "/tmp/oidserver-sessions",
'tools.sessions.timeout': 60,
'tools.sessions.name': 'oidserver_sid'
})
def testD(self):
raise nose.SkipTest
config = self.testInit.getConfiguration()
config.component_("TestComponent")
config.TestComponent.logLevel = 'INFO'
config.section_("General")
self.tempDir = self.testInit.generateWorkDir(config)
# try starting a component as a daemon:
config.TestComponent.componentDir = os.path.join( \
self.tempDir, "Components/TestComponent2")
os.makedirs(config.TestComponent.componentDir)
testComponent = TestComponent(config)
# we set the parent to true as we are testing
testComponent.startDaemon(keepParent=True)
time.sleep(2)
daemonFile = os.path.join(config.TestComponent.componentDir,
"Daemon.xml")
details = Details(daemonFile)
print('Is component alive: ' + str(details.isAlive()))
# create msgService to send stop message.
myThread = threading.currentThread()
factory = WMFactory("msgService", "WMCore.MsgService." + \
myThread.dialect)
myThread.transaction = Transaction(myThread.dbi)
msgService = factory.loadObject("MsgService")
msgService.registerAs("HarnessTest")
myThread.transaction.commit()
print(
'Publish a stop message to test if the component shutsdown gracefully'
)
myThread.transaction.begin()
msg = {'name': 'Stop', 'payload': ''}
msgService.publish(msg)
myThread.transaction.commit()
msgService.finish()
while details.isAlive():
print('Component has not received stop message')
time.sleep(2)
print('Daemon shutdown gracefully')
def findAlgos(self, dataset):
"""
Call to findAlgos
"""
myThread = threading.currentThread()
existingTransaction = self.beginTransaction()
factory = WMFactory("dbsUpload", "WMComponent.DBSUpload.Database."+ \
myThread.dialect)
findAlgos = factory.loadObject("FindAlgos")
# Add the file to the buffer (API Call)
#results = findFiles.execute(conn = self.getDBConn(), transaction=self.existingTransaction())
results = findAlgos.execute(datasetInfo=dataset,
conn=self.getDBConn(),
transaction=self.existingTransaction())
self.commitTransaction(existingTransaction)
return results
class TransferDaemon(BaseWorkerThread):
"""
_TransferDaemon_
Call multiprocessing library to instantiate a TransferWorker for each user.
"""
def __init__(self, config):
"""
Initialise class members
"""
#Need a better way to test this without turning off this next line
BaseWorkerThread.__init__(self)
#logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
#self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
self.config = config.AsyncTransfer
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.logger.debug('Connected to CouchDB')
self.pool = Pool(processes=self.config.pool_size)
try:
self.phedex = PhEDEx(responseType='xml')
except Exception, e:
self.logger.exception('PhEDEx exception: %s' % e)
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.schedAlgoDir,
namespace=self.config.schedAlgoDir)
result_list = []
current_running = []
