def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
"WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
#
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
logging.info("Setting the replication to central monitor ...")
self.localSummaryCouchDB.replicate(self.config.AnalyticsDataCollector.centralWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
class AnalyticsPoller(BaseWorkerThread):
"""
Gether the summary data for request (workflow) from local queue,
local job couchdb, wmbs/boss air and populate summary db for monitoring
"""
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = (
config.AnalyticsDataCollector.summaryLevel).lower()
self.pluginName = getattr(config.AnalyticsDataCollector, "pluginName",
None)
self.plugin = None
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(
self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(
self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi,
myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(
self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(
centralRequestCouchDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(
self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory(
"plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname=self.pluginName)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite(
)
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB(
)
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = {}
if not hasattr(self.config, "Tier0Feeder"):
localQInfo = self.localQueue.getAnalyticsData()
else:
logging.debug("Tier-0 instance, not checking WorkQueue")
# combine all the data from 3 sources
logging.info(
"""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ...""" %
(len(jobInfoFromCouch), len(fwjrInfoFromCouch),
len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch,
batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData,
localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info(
"%s requests Data combined,\n uploading request data..." %
len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests,
fwjrInfoFromCouch,
finishedTasks,
self.agentInfo, uploadTime,
self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB,
self.centralRequestCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info(
"Request data upload success\n %s request, \nsleep for next cycle"
% len(requestDocs))
DataUploadTime.setInfo(uploadTime, "ok")
except Exception as ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
class AnalyticsPoller(BaseWorkerThread):
"""
Gether the summary data for request (workflow) from local queue,
local job couchdb, wmbs/boss air and populate summary db for monitoring
"""
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
self.pluginName = getattr(config.AnalyticsDataCollector, "pluginName", None)
self.plugin = None
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
# use local db for tier0
if hasattr(self.config, "Tier0Feeder"):
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.General.centralWMStatsURL)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
self.dbsBufferUtil = DBSBufferUtil()
if self.pluginName is not None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname=self.pluginName)
@timeFunction
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
# jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
# fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
skippedInfoFromCouch = self.localCouchDB.getSkippedFilesSummaryByWorkflow()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
logging.info("Getting DBS PhEDEx upload status ...")
completedWfs = self.dbsBufferUtil.getPhEDExDBSStatusForCompletedWorkflows(summary=True)
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = {}
if not hasattr(self.config, "Tier0Feeder"):
localQInfo = self.localQueue.getAnalyticsData()
else:
logging.debug("Tier-0 instance, not checking WorkQueue")
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
WorkflowsWithSkippedFile(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s)
Completed workflows(%s).. ...""",
len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(skippedInfoFromCouch),
len(batchJobInfo), len(finishedTasks), len(localQInfo), len(completedWfs))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
tempCombinedData2 = combineAnalyticsData(tempCombinedData, localQInfo)
combinedRequests = combineAnalyticsData(tempCombinedData2, completedWfs)
# set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data...", len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
skippedInfoFromCouch, self.agentInfo,
uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
existingDocs = self.centralWMStatsCouchDB.getAllAgentRequestRevByID(self.agentInfo["agent_url"])
self.centralWMStatsCouchDB.bulkUpdateData(requestDocs, existingDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle", len(requestDocs))
self.centralWMStatsCouchDB.updateAgentInfoInPlace(self.agentInfo["agent_url"],
{"data_last_update": uploadTime, "data_error": "ok"})
except Exception as ex:
msg = str(ex)
logging.exception("Error occurred, will retry later: %s", msg)
try:
self.centralWMStatsCouchDB.updateAgentInfoInPlace(self.agentInfo["agent_url"], {"data_error": msg})
except:
logging.error("upload Agent Info to central couch failed")
class AnalyticsPoller(BaseWorkerThread):
"""
Gether the summary data for request (workflow) from local queue,
local job couchdb, wmbs/boss air and populate summary db for monitoring
"""
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
self.pluginName = getattr(config.AnalyticsDataCollector, "pluginName", None)
self.plugin = None
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = {}
if not hasattr(self.config, "Tier0Feeder"):
localQInfo = self.localQueue.getAnalyticsData()
else:
logging.debug("Tier-0 instance, not checking WorkQueue")
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
self.agentInfo, uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
DataUploadTime.setInfo(uploadTime, "ok")
except Exception as ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
DataUploadTime.setInfo(False, str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
class AnalyticsPoller(BaseWorkerThread):
"""
Gether the summary data for request (workflow) from local queue,
local job couchdb, wmbs/boss air and populate summary db for monitoring
"""
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
self.agentInfo = {}
self.agentInfo['agent_team'] = config.Agent.teamName
self.agentInfo['agent'] = config.Agent.agentName
# temporarly add port for the split test
self.agentInfo['agent_url'] = ("%s:%s" % (config.Agent.hostName, config.WMBSService.Webtools.port))
# need to get campaign, user, owner info
self.agentDocID = "agent+hostname"
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gether information
"""
#
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
logging.info("Setting the replication to central monitor ...")
self.localSummaryCouchDB.replicate(self.config.AnalyticsDataCollector.centralWMStatsURL)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getEventSummaryByWorkflow()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = self.localQueue.getAnalyticsData()
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch,
self.agentInfo, uploadTime, self.summaryLevel)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request \n uploading agent data" % len(requestDocs))
#TODO: agent info (need to include job Slots for the sites)
agentInfo = self.wmagentDB.getHeartBeatWarning()
agentInfo.update(self.agentInfo)
agentDocs = convertToAgentCouchDoc(agentInfo, self.config.ACDC, uploadTime)
self.localSummaryCouchDB.updateAgentInfo(agentDocs)
logging.info("Agent data upload success\n %s request" % len(agentDocs))
except Exception, ex:
logging.error(str(ex))
raise
class AnalyticsPoller(BaseWorkerThread):
"""
Gether the summary data for request (workflow) from local queue,
local job couchdb, wmbs/boss air and populate summary db for monitoring
"""
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
self.agentInfo = {}
self.agentInfo['agent_team'] = config.Agent.teamName
self.agentInfo['agent'] = config.Agent.agentName
# temporarly add port for the split test
self.agentInfo['agent_url'] = ("%s:%s" % (config.Agent.hostName, config.WMBSService.Webtools.port))
# need to get campaign, user, owner info
self.agentDocID = "agent+hostname"
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
self.pluginName = getattr(config.AnalyticsDataCollector, "pluginName", None)
self.plugin = None
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.centralWMStatsURL)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
logging.info("Getting Agent info ...")
agentInfo = self.collectAgentInfo()
#jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
#fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
#fwjrInfoFromCouch = self.localCouchDB.getEventSummaryByWorkflow()
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSite()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = self.localQueue.getAnalyticsData()
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s) ..."""
% (len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(batchJobInfo), len(finishedTasks), len(localQInfo)))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
combinedRequests = combineAnalyticsData(tempCombinedData, localQInfo)
#set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
self.uploadAgentInfoToCentralWMStats(agentInfo, uploadTime)
logging.info("%s requests Data combined,\n uploading request data..." % len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
self.agentInfo, uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralWMStatsCouchDB)
self.localSummaryCouchDB.uploadData(requestDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle" % len(requestDocs))
except Exception, ex:
logging.error("Error occurred, will retry later:")
logging.error(str(ex))
logging.error("Trace back: \n%s" % traceback.format_exc())
