def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(
**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.requestLocalCouchDB = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralCouchDBWriter = self.requestLocalCouchDB
else:
self.centralCouchDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.centralRequestDBURL)
self.reqmgr2Svc = ReqMgr(
self.config.TaskArchiver.ReqMgr2ServiceURL)
#TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
self.reqmgrSvc = RequestManager(
{'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
#Load the cleanout state ID and save it
stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
self.stateID = stateIDDAO.execute("cleanout")
return
def setup(self, parameters):
"""
Called at startup
"""
# set the connection for local couchDB call
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
self.archiveDelayHours = getattr(self.config.TaskArchiver,
'archiveDelayHours', 0)
self.wmstatsCouchDB = WMStatsWriter(
self.config.TaskArchiver.localWMStatsURL, "WMStatsAgent")
#TODO: we might need to use local db for Tier0
self.centralRequestDBReader = RequestDBReader(
self.config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
if self.useReqMgrForCompletionCheck:
self.deletableState = "announced"
self.centralRequestDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.centralRequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
if self.config.TaskArchiver.reqmgr2Only:
self.reqmgr2Svc = ReqMgr(
self.config.TaskArchiver.ReqMgr2ServiceURL)
else:
#TODO: remove this for reqmgr2
self.reqmgrSvc = RequestManager(
{'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
else:
# Tier0 case
self.deletableState = "completed"
# use local for update
self.centralRequestDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
jobDBurl = sanitizeURL(self.config.JobStateMachine.couchurl)['url']
jobDBName = self.config.JobStateMachine.couchDBName
self.jobCouchdb = CouchServer(jobDBurl)
self.jobsdatabase = self.jobCouchdb.connectDatabase("%s/jobs" %
jobDBName)
self.fwjrdatabase = self.jobCouchdb.connectDatabase("%s/fwjrs" %
jobDBName)
statSummaryDBName = self.config.JobStateMachine.summaryStatsDBName
self.statsumdatabase = self.jobCouchdb.connectDatabase(
statSummaryDBName)
def __init__(self, app, api, config, mount):
# main CouchDB database where requests/workloads are stored
RESTEntity.__init__(self, app, api, config, mount)
self.reqmgr_db = api.db_handler.get_db(config.couch_reqmgr_db)
self.reqmgr_db_service = RequestDBWriter(self.reqmgr_db, couchapp="ReqMgr")
# this need for the post validtiaon
self.gq_service = WorkQueue(config.couch_host, config.couch_workqueue_db)
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
"WMStatsAgent")
if hasattr(self.config, "Tier0Feeder"):
#use local db for tier0
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp = self.config.AnalyticsDataCollector.RequestCouchApp)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
if self.pluginName != None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname = self.pluginName)
def setUp(self):
"""
setup for test.
"""
myThread = threading.currentThread()
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection(destroyAllDatabase=True)
self.testInit.setSchema(
customModules=["WMCore.WMBS", "WMComponent.DBS3Buffer"],
useDefault=False)
self.databaseName = "taskarchiver_t_0"
self.testInit.setupCouch("%s/workloadsummary" % self.databaseName,
"WorkloadSummary")
self.testInit.setupCouch("%s/jobs" % self.databaseName, "JobDump")
self.testInit.setupCouch("%s/fwjrs" % self.databaseName, "FWJRDump")
self.testInit.setupCouch("wmagent_summary_t", "WMStats")
self.testInit.setupCouch("wmagent_summary_central_t", "WMStats")
self.testInit.setupCouch("stat_summary_t", "SummaryStats")
reqmgrdb = "reqmgrdb_t"
self.testInit.setupCouch(reqmgrdb, "ReqMgr")
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqmgrdb)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestWriter.defaultStale = {}
self.daofactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getJobs = self.daofactory(classname="Jobs.GetAllJobs")
self.inject = self.daofactory(
classname="Workflow.MarkInjectedWorkflows")
self.testDir = self.testInit.generateWorkDir()
os.makedirs(os.path.join(self.testDir, 'specDir'))
self.nJobs = 10
self.campaignName = 'aCampaign'
self.uploadPublishInfo = False
self.uploadPublishDir = None
return
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["ReqMgr"]
self.testInit = TestInitCouchApp('RequestDBServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'requsetdb_t'
self.testInit.setupCouch(dbName, *self.couchApps)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, dbName)
self.requestWriter = RequestDBWriter(reqDBURL)
self.requestReader = RequestDBReader(reqDBURL)
self.requestWriter.defaultStale = {}
self.requestReader.defaultStale = {}
return
def moveToArchived(self, config):
"""
gather active data statistics
"""
testbedWMStats = WMStatsReader(config.wmstats_url,
reqdbURL=config.reqmgrdb_url)
reqdbWriter = RequestDBWriter(config.reqmgrdb_url)
statusTransition = {
"aborted": ["aborted-completed", "aborted-archived"],
"rejected": ["rejected-archived"]
}
for status, nextStatusList in statusTransition.items():
requests = testbedWMStats.getRequestByStatus([status],
jobInfoFlag=True,
legacyFormat=True)
self.logger.info("checking %s workflows: %d" %
(status, len(requests)))
if len(requests) > 0:
requestCollection = RequestInfoCollection(requests)
requestsDict = requestCollection.getData()
numOfArchived = 0
for requestName, requestInfo in requestsDict.items():
if requestInfo.getJobSummary().getTotalJobs() == 0:
for nextStatus in nextStatusList:
reqdbWriter.updateRequestStatus(
requestName, nextStatus)
numOfArchived += 1
self.logger.info("Total %s-archieved: %d" %
(status, numOfArchived))
return
def setUp(self):
"""
_setUp_
"""
self.schema = []
self.couchApps = ["WMStats"]
self.testInit = TestInitCouchApp('WorkQueueServiceTest')
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=self.schema, useDefault=False)
dbName = 'wmstats_t'
self.testInit.setupCouch(dbName, "WMStats")
reqDBName = "reqmgrdb_t"
self.testInit.setupCouch(reqDBName, "ReqMgr")
wmstatsURL = "%s/%s" % (self.testInit.couchUrl, dbName)
reqDBURL = "%s/%s" % (self.testInit.couchUrl, reqDBName)
self.reqDBWriter = RequestDBWriter(reqDBURL)
self.wmstatsReader = WMStatsReader(wmstatsURL, reqdbURL=reqDBURL)
self.wmstatsReader.defaultStale = {}
self.wmstatsReader.reqDB.defaultStale = {}
return
def setUp(self):
"""
_setUp_
Setup the test environment
"""
self.testInit = TestInit(__file__)
self.testInit.setDatabaseConnection()
self.testInit.setSchema(["WMCore.WMBS"])
self.requestCouchDB = 'wmstats_plugin_t'
self.testInit.setupCouch(self.requestCouchDB, 'T0Request')
self.testDir = self.testInit.generateWorkDir()
reqDBURL = "%s/%s" % (os.environ['COUCHURL'], self.requestCouchDB)
self.requestDBWriter = RequestDBWriter(reqDBURL, couchapp="T0Request")
self.requestDBWriter._setNoStale()
self.stateMap = {}
self.orderedStates = []
self.plugin = None
return
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
if self.transferSystemBaseDir != None:
if not os.path.exists(self.transferSystemBaseDir):
self.transferSystemBaseDir = None
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy", None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp = config.AnalyticsDataCollector.RequestCouchApp)
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl = hltConfConnectUrl, options = {})
dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname = "RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl = storageManagerConnectUrl, options = {})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl = popConLogConnectUrl, options = {})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname = "Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase, "connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging, dburl = t0datasvcConnectUrl, options = {})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(package = "T0.WMBS",
logger = logging,
dbinterface = dbInterfaceT0DataSvc)
return
def __init__(self, rest, config):
super(StepChainParentageFixTask, self).__init__(config)
self.reqmgrDB = RequestDBWriter(config.reqmgrdb_url)
self.dbsSvc = DBS3Reader(config.dbs_url, logger=self.logger)
self.statusToCheck = ["closed-out", "announced", "normal-archived"]
def main():
BASE_URL = "https://cmsweb.cern.ch"
COUCH_URL = "%s/couchdb" % BASE_URL
dbs_url = "https://cmsweb.cern.ch/dbs/prod/global/DBSWriter"
reqmgrdb_url = "%s/reqmgr_workload_cache" % (COUCH_URL)
statusToCheck = ['closed-out', 'announced']
logger = setupLogger()
dbsSvc = DBS3Reader(dbs_url, logger=logger)
reqmgrDB = RequestDBWriter(reqmgrdb_url)
logger.info("Running fixStepChainParentage thread for statuses")
childDatasets = set()
requests = set()
requestsByChildDataset = {}
for status in statusToCheck:
reqByChildDS = getChildDatasetsForStepChainMissingParent(
reqmgrDB, status, logger)
logger.info("Retrieved %d datasets to fix parentage, in status: %s",
len(reqByChildDS), status)
childDatasets = childDatasets.union(set(reqByChildDS.keys()))
# We need to just get one of the StepChain workflow if multiple workflow contains the same datasets. (i.e. ACDC)
requestsByChildDataset.update(reqByChildDS)
for wfs in reqByChildDS.values():
requests = requests.union(wfs)
logger.info(" datasets are: %s", reqByChildDS)
failedRequests = set()
totalChildDS = len(childDatasets)
fixCount = 0
for childDS in childDatasets:
logger.info("Resolving parentage for dataset: %s", childDS)
try:
failedBlocks = dbsSvc.fixMissingParentageDatasets(childDS,
insertFlag=True)
except Exception as exc:
logger.exception(
"Failed to resolve parentage data for dataset: %s. Error: %s",
childDS, str(exc))
failedRequests = failedRequests.union(
requestsByChildDataset[childDS])
else:
if failedBlocks:
logger.warning(
"These blocks failed to be resolved and will be retried later: %s",
failedBlocks)
failedRequests = failedRequests.union(
requestsByChildDataset[childDS])
else:
fixCount += 1
logger.info(
"Parentage for '%s' successfully updated. Processed %s out of %s datasets.",
childDS, fixCount, totalChildDS)
logger.info(" dataset sorted: %s\n", childDS)
requestsToUpdate = requests - failedRequests
### FIXME: disable the block below if you do NOT want to update the
# workflow in ReqMgr2
for request in requestsToUpdate:
try:
reqmgrDB.updateRequestProperty(request,
{"ParentageResolved": True})
logger.info("Marked ParentageResolved=True for request: %s",
request)
except Exception as exc:
logger.error(
"Failed to update 'ParentageResolved' flag to True for request: %s",
request)
msg = "A total of %d requests have been processed, where %d will have to be retried in the next cycle."
logger.info(msg, len(requestsToUpdate), len(failedRequests))
sys.exit(0)
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="T0.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy",
None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(
config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=config.AnalyticsDataCollector.RequestCouchApp)
self.injectedRuns = set()
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging,
dburl=hltConfConnectUrl,
options={})
self.dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package="T0.WMBS",
logger=logging,
dbinterface=self.dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(
classname="RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging,
dburl=storageManagerConnectUrl,
options={})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.dbInterfaceSMNotify = None
if hasattr(config, "SMNotifyDatabase"):
smNotifyConnectUrl = config.SMNotifyDatabase.connectUrl
dbFactorySMNotify = DBFactory(logging,
dburl=smNotifyConnectUrl,
options={})
self.dbInterfaceSMNotify = dbFactorySMNotify.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase,
"connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging,
dburl=popConLogConnectUrl,
options={})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(
package="T0.WMBS",
logger=logging,
dbinterface=dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(
classname="Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase,
"connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging,
dburl=t0datasvcConnectUrl,
options={})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(
package="T0.WMBS",
logger=logging,
dbinterface=dbInterfaceT0DataSvc)
#
# Set deployment ID
#
SetDeploymentIdDAO = self.daoFactory(
classname="Tier0Feeder.SetDeploymentID")
GetDeploymentIdDAO = self.daoFactory(
classname="Tier0Feeder.GetDeploymentID")
try:
self.deployID = GetDeploymentIdDAO.execute()
if self.deployID == 0:
self.deployID = int(
datetime.datetime.now().strftime("%y%m%d%H%M%S"))
SetDeploymentIdDAO.execute(self.deployID)
except:
logging.exception(
"Something went wrong with setting deployment ID")
raise
return
