def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.group = getattr(config.PhEDExInjector, "group", "DataOps")
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
"srm-cms-disk.gridpp.rl.ac.uk"])
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "PhEDExInjector")
self.filesToRecover = None
def __init__(self, config):
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
#DAO factory for WMBS objects
self.daoFactory = DAOFactory(package = "WMCore.WMBS", \
logger = logging,
dbinterface = myThread.dbi)
self.config = config
#Libraries
self.resourceControl = ResourceControl()
self.changeState = ChangeState(self.config)
self.repollCount = getattr(self.config.JobSubmitter, 'repollCount',
10000)
# BossAir
self.bossAir = BossAirAPI(config=self.config)
# Additions for caching-based JobSubmitter
self.workflowTimestamps = {}
self.cachedJobIDs = set()
self.cachedJobs = {}
self.jobDataCache = {}
self.jobsToPackage = {}
self.sandboxPackage = {}
self.siteKeys = {}
self.locationDict = {}
self.cmsNames = {}
self.drainSites = []
self.sortedSites = []
self.packageSize = getattr(self.config.JobSubmitter, 'packageSize',
500)
self.collSize = getattr(self.config.JobSubmitter, 'collectionSize',
self.packageSize * 1000)
# initialize the alert framework (if available)
self.initAlerts(compName="JobSubmitter")
try:
if not getattr(self.config.JobSubmitter, 'submitDir', None):
self.config.JobSubmitter.submitDir = self.config.JobSubmitter.componentDir
self.packageDir = os.path.join(self.config.JobSubmitter.submitDir,
'packages')
if not os.path.exists(self.packageDir):
os.makedirs(self.packageDir)
except Exception, ex:
msg = "Error while trying to create packageDir %s\n!"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
try:
logging.debug("PackageDir: %s" % self.packageDir)
logging.debug("Config: %s" % config)
except:
pass
raise JobSubmitterPollerException(msg)
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI()
def __init__(self, config):
"""
Initialise class members
"""
# Need a better way to test this without turning off this next line
BaseWorkerThread.__init__(self)
# logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
# self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
self.config = config.AsyncTransfer
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug("Configuration loaded")
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.logger.debug("Connected to CouchDB")
self.pool = Pool(processes=self.config.pool_size)
try:
self.phedex = PhEDEx(responseType="xml")
except Exception, e:
self.logger.exception("PhEDEx exception: %s" % e)
def __init__(self, config):
"""
Initialise class members
"""
# Used for creating connections/transactions
myThread = threading.currentThread()
self.threadID = myThread.ident
self.dbi = myThread.dbi
self.logger = myThread.logger
BaseWorkerThread.__init__(self)
# get the db owner
self.config = config
dbconfig = config.section_("CoreDatabase")
self.dbowner=dbconfig.dbowner
self.newBlock = False
self.datasetCache = {}
self.datasetCache['primDs']={}
self.datasetCache['dataset']={}
self.datasetCache['acquisitionEra']={}
self.datasetCache['processingVersion']={}
self.datasetCache['phyGrp']={}
self.datasetCache['dataTier']={}
self.datasetCache['primDsTp']={}
self.datasetCache['datasetAccTp']={}
self.datasetCache['processedDs']={}
self.datasetCache['relVer']={}
self.datasetCache['pHash']={}
self.datasetCache['appExe']={}
self.datasetCache['conf']={} #key=app:rel:phash
def __init__(self, config, phedex, nodeMappings):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.phedex = phedex
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.group = getattr(config.PhEDExInjector, "group", "DataOps")
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
for node in nodeMappings["phedex"]["node"]:
if node["kind"] not in self.seMap:
self.seMap[node["kind"]] = {}
logging.info("Adding mapping %s -> %s", node["se"], node["name"])
self.seMap[node["kind"]][node["se"]] = node["name"]
self.nodeNames.append(node["name"])
self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
"srm-cms-disk.gridpp.rl.ac.uk"])
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "PhEDExInjector")
self.blocksToRecover = []
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
self.changeState = ChangeState(self.config)
self.maxRetries = self.config.ErrorHandler.maxRetries
if not isinstance(self.maxRetries, dict):
self.maxRetries = {"default": self.maxRetries}
if "default" not in self.maxRetries:
raise ErrorHandlerException("Max retries for the default job type must be specified")
self.maxProcessSize = getattr(self.config.ErrorHandler, "maxProcessSize", 250)
self.exitCodes = getattr(self.config.ErrorHandler, "failureExitCodes", [])
self.maxFailTime = getattr(self.config.ErrorHandler, "maxFailTime", 32 * 3600)
self.readFWJR = getattr(self.config.ErrorHandler, "readFWJR", False)
self.passCodes = getattr(self.config.ErrorHandler, "passExitCodes", [])
self.getJobs = self.daoFactory(classname="Jobs.GetAllJobs")
self.idLoad = self.daoFactory(classname="Jobs.LoadFromIDWithType")
self.loadAction = self.daoFactory(classname="Jobs.LoadForErrorHandler")
self.dataCollection = DataCollectionService(url=config.ACDC.couchurl, database=config.ACDC.database)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.changeState = ChangeState(self.config)
self.maxRetries = self.config.ErrorHandler.maxRetries
if not isinstance(self.maxRetries, dict):
self.maxRetries = {'default' : self.maxRetries}
if 'default' not in self.maxRetries:
raise ErrorHandlerException('Max retries for the default job type must be specified')
self.maxProcessSize = getattr(self.config.ErrorHandler, 'maxProcessSize', 250)
self.exitCodes = getattr(self.config.ErrorHandler, 'failureExitCodes', [])
self.maxFailTime = getattr(self.config.ErrorHandler, 'maxFailTime', 32 * 3600)
self.readFWJR = getattr(self.config.ErrorHandler, 'readFWJR', False)
self.passCodes = getattr(self.config.ErrorHandler, 'passExitCodes', [])
self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")
self.idLoad = self.daoFactory(classname = "Jobs.LoadFromIDWithType")
self.loadAction = self.daoFactory(classname = "Jobs.LoadForErrorHandler")
self.dataCollection = DataCollectionService(url = config.ACDC.couchurl,
database = config.ACDC.database)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
pluginPath = getattr(self.config.RetryManager, "pluginPath",
"WMComponent.RetryManager.PlugIns")
self.pluginFactory = WMFactory("plugins", pluginPath)
self.changeState = ChangeState(self.config)
self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")
# initialize the alert framework (if available) (self.sendAlert())
self.initAlerts(compName = "RetryManager")
try:
pluginName = getattr(self.config.RetryManager, 'pluginName', 'DefaultRetryAlgo')
self.plugin = self.pluginFactory.loadObject(classname = pluginName,
args = config)
except Exception, ex:
msg = "Error loading plugin %s on path %s\n" % (pluginName, pluginPath)
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
raise RetryManagerException(msg)
def __init__(self, config):
"""
Initialize config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.changeState = ChangeState(self.config)
self.maxRetries = self.config.ErrorHandler.maxRetries
self.maxProcessSize = getattr(self.config.ErrorHandler, 'maxProcessSize', 250)
self.getJobs = self.daoFactory(classname = "Jobs.GetAllJobs")
self.dataCollection = DataCollectionService(url = config.ACDC.couchurl,
database = config.ACDC.database)
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "ErrorHandler")
return
def __init__(self, resources):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.resources = resources
self.wq = localQueue()
def __init__(self, config, phedex, nodeMappings):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.phedex = phedex
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.group = getattr(config.PhEDExInjector, "group", "DataOps")
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
for node in nodeMappings["phedex"]["node"]:
if node["kind"] not in self.seMap:
self.seMap[node["kind"]] = {}
logging.info("Adding mapping %s -> %s", node["se"], node["name"])
self.seMap[node["kind"]][node["se"]] = node["name"]
self.nodeNames.append(node["name"])
self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
"srm-cms-disk.gridpp.rl.ac.uk"])
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "PhEDExInjector")
self.blocksToRecover = []
def __init__(self, config):
"""
Initialize config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.phedexGroup = config.PhEDExInjector.phedexGroup
self.pollCounter = 0
self.subFrequency = None
if getattr(config.PhEDExInjector, "subscribeDatasets", False):
pollInterval = config.PhEDExInjector.pollInterval
subInterval = config.PhEDExInjector.subscribeInterval
self.subFrequency = max(1, int(round(subInterval / pollInterval)))
logging.info("SubscribeDataset and deleteBlocks will run every %d polling cycles", self.subFrequency)
# subscribe on first cycle
self.pollCounter = self.subFrequency - 1
# retrieving the node mappings is fickle and can fail quite often
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
"json", dbsUrl=self.dbsUrl)
try:
nodeMappings = self.phedex.getNodeMap()
except:
time.sleep(2)
try:
nodeMappings = self.phedex.getNodeMap()
except:
time.sleep(4)
nodeMappings = self.phedex.getNodeMap()
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
for node in nodeMappings["phedex"]["node"]:
if node["kind"] not in self.seMap:
self.seMap[node["kind"]] = {}
logging.info("Adding mapping %s -> %s", node["se"], node["name"])
self.seMap[node["kind"]][node["se"]] = node["name"]
self.nodeNames.append(node["name"])
self.phedexNodes = {'MSS': [], 'Disk': []}
for node in nodeMappings["phedex"]["node"]:
if node["kind"] in ["MSS", "Disk"]:
self.phedexNodes[node["kind"]].append(node["name"])
self.blocksToRecover = []
# X-component configuration is BAD! But it will only be here during the
# Rucio commissioning within WM
self.listTiersToSkip = config.RucioInjector.listTiersToInject
logging.info("Component configured to skip data injection for data tiers: %s",
self.listTiersToSkip)
return
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
self.siteDB = SiteDBJSON()
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.group = getattr(config.PhEDExInjector, "group", "DataOps")
self.safeMode = getattr(config.PhEDExInjector, "safeOperationMode", False)
# Subscribed state in the DBSBuffer table for datasets
self.terminalSubscriptionState = 1
if self.safeMode:
self.terminalSubscriptionState = 2
# We will map node names to CMS names, that what the spec will have.
# If a CMS name is associated to many PhEDEx node then choose the MSS option
self.cmsToPhedexMap = {}
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "PhEDExInjector")
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.AsyncTransfer
# self.logger is set up by the BaseWorkerThread, we just set it's level
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.pluginDir, namespace = self.config.pluginDir)
# Asynch db
server = CouchServer(dburl=self.config.couch_instance, ckey=self.config.opsProxy, cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
self.logger.debug('Connected to CouchDB')
return
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config
self.accountantWorkSize = getattr(self.config.JobAccountant,
'accountantWorkSize', 100)
return
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = config.AnalyticsDataCollector.summaryLevel
proxyArgs = {'logger': logging.getLogger(), 'cleanEnvironment': True}
self.proxy = Proxy(proxyArgs)
self.proxyFile = self.proxy.getProxyFilename() # X509_USER_PROXY
self.userCertFile = self.proxy.getUserCertFilename() # X509_USER_CERT
# credential lifetime warning/error thresholds, in days
self.credThresholds = {'proxy': {'error': 3, 'warning': 5},
'certificate': {'error': 10, 'warning': 20}}
# Monitoring setup
self.userAMQ = getattr(config.AgentStatusWatcher, "userAMQ", None)
self.passAMQ = getattr(config.AgentStatusWatcher, "passAMQ", None)
self.postToAMQ = getattr(config.AgentStatusWatcher, "enableAMQ", False)
self.topicAMQ = getattr(config.AgentStatusWatcher, "topicAMQ", None)
self.hostPortAMQ = getattr(config.AgentStatusWatcher, "hostPortAMQ", [('cms-mb.cern.ch', 61313)])
# T0 doesn't have WorkQueue, so some monitoring/replication code has to be skipped here
if hasattr(self.config, "Tier0Feeder"):
self.isT0agent = True
self.producer = "tier0wmagent"
else:
self.isT0agent = False
self.producer = "wmagent"
localWQUrl = config.AnalyticsDataCollector.localQueueURL
self.workqueueDS = WorkQueueDS(localWQUrl)
def __init__(self, resources):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.resources = resources
self.wq = localQueue()
def __init__(self, config):
"""
Initialise class members
"""
#Need a better way to test this without turning off this next line
BaseWorkerThread.__init__(self)
#logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
#self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
self.config = config.AsyncTransfer
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.pool = Pool(processes=self.config.pool_size)
# Set up a factory for loading plugins
self.factory = WMFactory(self.config.schedAlgoDir, namespace = self.config.schedAlgoDir)
self.dropbox_dir = '%s/dropbox/inputs' % self.config.componentDir
if not os.path.isdir(self.dropbox_dir):
try:
os.makedirs(self.dropbox_dir)
except OSError, e:
if e.errno == errno.EEXIST:
pass
else:
self.logger.error('Unknown error in mkdir' % e.errno)
raise
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.group = getattr(config.PhEDExInjector, "group", "DataOps")
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
self.diskSites = getattr(config.PhEDExInjector, "diskSites", ["storm-fe-cms.cr.cnaf.infn.it",
"srm-cms-disk.gridpp.rl.ac.uk"])
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "PhEDExInjector")
self.filesToRecover = None
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.Statistics
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
server = CouchServer(
dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(
dburl=self.config.config_couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.logger.debug('Connected to CouchDB')
statserver = CouchServer(self.config.couch_statinstance)
self.statdb = statserver.connectDatabase(
self.config.statitics_database)
self.logger.debug('Connected to Stat CouchDB')
self.iteration_docs = []
self.exptime = None
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.FilesCleaner
self.logger.debug('Configuration loaded')
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(self.config.config_database)
self.logger.debug('Connected to files DB')
self.phedex = PhEDEx(responseType='xml')
self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
try:
os.makedirs(self.log_dir)
except OSError, e:
if e.errno == errno.EEXIST:
pass
else:
self.logger.error('Unknown error in mkdir' % e.errno)
raise
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
"json")
self.siteDB = SiteDBJSON()
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.group = getattr(config.PhEDExInjector, "group", "DataOps")
self.safeMode = getattr(config.PhEDExInjector, "safeOperationMode",
False)
self.replicaOnly = getattr(config.PhEDExInjector, "replicaOnly", False)
# Subscribed state in the DBSBuffer table for datasets
self.terminalSubscriptionState = 1
if self.safeMode:
self.terminalSubscriptionState = 2
# We will map node names to CMS names, that what the spec will have.
# If a CMS name is associated to many PhEDEx node then choose the MSS option
self.cmsToPhedexMap = {}
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName="PhEDExInjector")
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.Statistics
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.mon_db = server.connectDatabase(self.config.mon_database)
self.logger.debug('Connected to CouchDB')
statserver = CouchServer(self.config.couch_statinstance)
self.statdb = statserver.connectDatabase(
self.config.statitics_database)
self.logger.debug('Connected to Stat CouchDB')
self.iteration_docs = []
self.exptime = None
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
def __init__(self, config):
"""
Initialise class members
"""
#Need a better way to test this without turning off this next line
BaseWorkerThread.__init__(self)
#logging.basicConfig(format = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s',datefmt = '%m-%d %H:%M')
#self.logger = logging.getLogger()
# self.logger is set up by the BaseWorkerThread, we just set it's level
self.config = config.AsyncTransfer
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
server = CouchServer(dburl=self.config.couch_instance,
ckey=self.config.opsProxy,
cert=self.config.opsProxy)
self.db = server.connectDatabase(self.config.files_database)
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.logger.debug('Connected to CouchDB')
self.pool = Pool(processes=self.config.pool_size)
try:
self.phedex = PhEDEx(responseType='xml')
except Exception, e:
self.logger.exception('PhEDEx exception: %s' % e)
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config.FilesCleaner
self.logger.debug('Configuration loaded')
try:
self.logger.setLevel(self.config.log_level)
except:
import logging
self.logger = logging.getLogger()
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
config_server = CouchServer(dburl=self.config.config_couch_instance)
self.config_db = config_server.connectDatabase(
self.config.config_database)
self.logger.debug('Connected to files DB')
self.phedex = PhEDEx(responseType='xml')
self.log_dir = '%s/logs/%s/%s/%s' % ( self.config.componentDir, \
str(datetime.datetime.now().month), str(datetime.datetime.now().year), "Ops")
try:
os.makedirs(self.log_dir)
except OSError, e:
if e.errno == errno.EEXIST:
pass
else:
self.logger.error('Unknown error in mkdir' % e.errno)
raise
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = config.AnalyticsDataCollector.summaryLevel
proxyArgs = {'logger': logging.getLogger()}
self.proxy = Proxy(proxyArgs)
self.proxyFile = self.proxy.getProxyFilename() # X509_USER_PROXY
self.userCertFile = self.proxy.getUserCertFilename() # X509_USER_CERT
# credential lifetime warning/error thresholds, in days
self.credThresholds = {'proxy': {'error': 3, 'warning': 5},
'certificate': {'error': 10, 'warning': 20}}
# Monitoring setup
self.userAMQ = getattr(config.AgentStatusWatcher, "userAMQ", None)
self.passAMQ = getattr(config.AgentStatusWatcher, "passAMQ", None)
self.postToAMQ = getattr(config.AgentStatusWatcher, "enableAMQ", False)
self.topicAMQ = getattr(config.AgentStatusWatcher, "topicAMQ", None)
self.hostPortAMQ = getattr(config.AgentStatusWatcher, "hostPortAMQ", [('cms-mb.cern.ch', 61313)])
# T0 doesn't have WorkQueue, so some monitoring/replication code has to be skipped here
if hasattr(self.config, "Tier0Feeder"):
self.isT0agent = True
self.producer = "tier0wmagent"
else:
self.isT0agent = False
self.producer = "wmagent"
localWQUrl = config.AnalyticsDataCollector.localQueueURL
self.workqueueDS = WorkQueueDS(localWQUrl)
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
self.config = config
self.drainAPI = DrainStatusAPI()
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.pollCounter = 0
self.subFrequency = None
if getattr(config.PhEDExInjector, "subscribeDatasets", False):
pollInterval = config.PhEDExInjector.pollInterval
subInterval = config.PhEDExInjector.subscribeInterval
self.subFrequency = max(1, int(round(subInterval / pollInterval)))
logging.info(
"SubscribeDataset and deleteBlocks will run every %d polling cycles",
self.subFrequency)
# subscribe on first cycle
self.pollCounter = self.subFrequency - 1
# retrieving the node mappings is fickle and can fail quite often
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl},
"json")
try:
nodeMappings = self.phedex.getNodeMap()
except:
time.sleep(2)
try:
nodeMappings = self.phedex.getNodeMap()
except:
time.sleep(4)
nodeMappings = self.phedex.getNodeMap()
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
for node in nodeMappings["phedex"]["node"]:
if node["kind"] not in self.seMap:
self.seMap[node["kind"]] = {}
logging.info("Adding mapping %s -> %s", node["se"], node["name"])
self.seMap[node["kind"]][node["se"]] = node["name"]
self.nodeNames.append(node["name"])
self.phedexNodes = {'MSS': [], 'Disk': []}
for node in nodeMappings["phedex"]["node"]:
if node["kind"] in ["MSS", "Disk"]:
self.phedexNodes[node["kind"]].append(node["name"])
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName="PhEDExInjector")
self.blocksToRecover = []
return
def __init__(self, config):
"""
_init_
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=myThread.dbi)
self.tier0ConfigFile = config.Tier0Feeder.tier0ConfigFile
self.specDirectory = config.Tier0Feeder.specDirectory
self.dropboxuser = getattr(config.Tier0Feeder, "dropboxuser", None)
self.dropboxpass = getattr(config.Tier0Feeder, "dropboxpass", None)
self.transferSystemBaseDir = getattr(config.Tier0Feeder, "transferSystemBaseDir", None)
if self.transferSystemBaseDir != None:
if not os.path.exists(self.transferSystemBaseDir):
self.transferSystemBaseDir = None
self.dqmUploadProxy = getattr(config.Tier0Feeder, "dqmUploadProxy", None)
self.serviceProxy = getattr(config.Tier0Feeder, "serviceProxy", None)
self.localRequestCouchDB = RequestDBWriter(
config.AnalyticsDataCollector.localT0RequestDBURL, couchapp=config.AnalyticsDataCollector.RequestCouchApp
)
hltConfConnectUrl = config.HLTConfDatabase.connectUrl
dbFactoryHltConf = DBFactory(logging, dburl=hltConfConnectUrl, options={})
dbInterfaceHltConf = dbFactoryHltConf.connect()
daoFactoryHltConf = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=dbInterfaceHltConf)
self.getHLTConfigDAO = daoFactoryHltConf(classname="RunConfig.GetHLTConfig")
storageManagerConnectUrl = config.StorageManagerDatabase.connectUrl
dbFactoryStorageManager = DBFactory(logging, dburl=storageManagerConnectUrl, options={})
self.dbInterfaceStorageManager = dbFactoryStorageManager.connect()
self.getExpressReadyRunsDAO = None
if hasattr(config, "PopConLogDatabase"):
popConLogConnectUrl = getattr(config.PopConLogDatabase, "connectUrl", None)
if popConLogConnectUrl != None:
dbFactoryPopConLog = DBFactory(logging, dburl=popConLogConnectUrl, options={})
dbInterfacePopConLog = dbFactoryPopConLog.connect()
daoFactoryPopConLog = DAOFactory(package="T0.WMBS", logger=logging, dbinterface=dbInterfacePopConLog)
self.getExpressReadyRunsDAO = daoFactoryPopConLog(classname="Tier0Feeder.GetExpressReadyRuns")
self.haveT0DataSvc = False
if hasattr(config, "T0DataSvcDatabase"):
t0datasvcConnectUrl = getattr(config.T0DataSvcDatabase, "connectUrl", None)
if t0datasvcConnectUrl != None:
self.haveT0DataSvc = True
dbFactoryT0DataSvc = DBFactory(logging, dburl=t0datasvcConnectUrl, options={})
dbInterfaceT0DataSvc = dbFactoryT0DataSvc.connect()
self.daoFactoryT0DataSvc = DAOFactory(
package="T0.WMBS", logger=logging, dbinterface=dbInterfaceT0DataSvc
)
return
def __init__(self, config):
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
#DAO factory for WMBS objects
self.daoFactory = DAOFactory(package = "WMCore.WMBS", \
logger = logging,
dbinterface = myThread.dbi)
self.config = config
#Libraries
self.resourceControl = ResourceControl()
self.changeState = ChangeState(self.config)
self.repollCount = getattr(self.config.JobSubmitter, 'repollCount', 10000)
# BossAir
self.bossAir = BossAirAPI(config = self.config)
# Additions for caching-based JobSubmitter
self.workflowTimestamps = {}
self.cachedJobIDs = set()
self.cachedJobs = {}
self.jobDataCache = {}
self.jobsToPackage = {}
self.sandboxPackage = {}
self.siteKeys = {}
self.locationDict = {}
self.cmsNames = {}
self.drainSites = []
self.sortedSites = []
self.packageSize = getattr(self.config.JobSubmitter, 'packageSize', 500)
self.collSize = getattr(self.config.JobSubmitter, 'collectionSize',
self.packageSize * 1000)
# initialize the alert framework (if available)
self.initAlerts(compName = "JobSubmitter")
try:
if not getattr(self.config.JobSubmitter, 'submitDir', None):
self.config.JobSubmitter.submitDir = self.config.JobSubmitter.componentDir
self.packageDir = os.path.join(self.config.JobSubmitter.submitDir, 'packages')
if not os.path.exists(self.packageDir):
os.makedirs(self.packageDir)
except Exception, ex:
msg = "Error while trying to create packageDir %s\n!"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
try:
logging.debug("PackageDir: %s" % self.packageDir)
logging.debug("Config: %s" % config)
except:
pass
raise JobSubmitterPollerException(msg)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.changeState = ChangeState(self.config)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.loadAction = self.daoFactory(classname = "Jobs.LoadFromIDWithWorkflow")
# Variables
self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
"numberOfJobsToCluster", 1000)
# initialize the alert framework (if available)
self.initAlerts(compName = "JobArchiver")
try:
self.uploadPublishDir = getattr(self.config.JobArchiver, 'uploadPublishDir',
os.path.join(config.JobArchiver.componentDir, 'publishDir'))
self.logDir = getattr(config.JobArchiver, 'logDir',
os.path.join(config.JobArchiver.componentDir, 'logDir'))
if not os.path.isdir(self.logDir):
os.makedirs(self.logDir)
if not os.path.isdir(self.uploadPublishDir):
os.makedirs(self.uploadPublishDir)
except Exception as ex:
msg = "Unhandled exception while setting up logDir and/or uploadPublishDir!\n"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg = msg)
try:
logging.debug("Directory: %s" % self.logDir)
logging.debug("Config: %s" % config)
except:
pass
raise JobArchiverPollerException(msg)
try:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
except Exception as ex:
msg = "Could not load workQueue"
msg += str(ex)
logging.error(msg)
#raise JobArchiverPollerException(msg)
self.uploadPublishInfo = getattr(self.config.JobArchiver, 'uploadPublishInfo', False)
self.userFileCacheURL = getattr(self.config.JobArchiver, 'userFileCacheURL', None)
self.handleWorkflowInjection = getattr(self.config.JobArchiver,
'handleInjected', True)
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.config = config
self.changeState = ChangeState(self.config)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.loadAction = self.daoFactory(classname="Jobs.LoadFromIDWithWorkflow")
# Variables
self.numberOfJobsToCluster = getattr(self.config.JobArchiver,
"numberOfJobsToCluster", 1000)
# initialize the alert framework (if available)
self.initAlerts(compName="JobArchiver")
try:
self.uploadPublishDir = getattr(self.config.JobArchiver, 'uploadPublishDir',
os.path.join(config.JobArchiver.componentDir, 'publishDir'))
self.logDir = getattr(config.JobArchiver, 'logDir',
os.path.join(config.JobArchiver.componentDir, 'logDir'))
if not os.path.isdir(self.logDir):
os.makedirs(self.logDir)
if not os.path.isdir(self.uploadPublishDir):
os.makedirs(self.uploadPublishDir)
except Exception as ex:
msg = "Unhandled exception while setting up logDir and/or uploadPublishDir!\n"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
try:
logging.debug("Directory: %s", self.logDir)
logging.debug("Config: %s", config)
except:
pass
raise JobArchiverPollerException(msg)
try:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
except Exception as ex:
msg = "Could not load workQueue"
msg += str(ex)
logging.error(msg)
# raise JobArchiverPollerException(msg)
self.uploadPublishInfo = getattr(self.config.JobArchiver, 'uploadPublishInfo', False)
self.userFileCacheURL = getattr(self.config.JobArchiver, 'userFileCacheURL', None)
self.handleWorkflowInjection = getattr(self.config.JobArchiver,
'handleInjected', True)
return
def __init__(self, threads=8):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.fileset_watch = {}
self.last_poll_time = time.time() / 60
self.workq = ThreadPool \
([self.pollExternal for _ in range(threads)])
def __init__(self, threads=8):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.fileset_watch = {}
self.last_poll_time = time.time()/60
self.workq = ThreadPool \
([self.pollExternal for _ in range(threads)])
def __init__(self, config):
"""
Initialize
"""
BaseWorkerThread.__init__(self)
self.config = config
self.ssb2AgentStatus = {
'enabled': 'Normal',
'drain': 'Draining',
'disabled': 'Down',
'test': 'Draining',
'unknown': None
}
self.tasksCPU = ['Processing', 'Production']
self.tasksIO = ['Merge', 'Cleanup', 'Harvesting', 'LogCollect', 'Skim']
self.minCPUSlots = 50
self.minIOSlots = 25
# get dashboard url, set metric columns from config
_token = config.AgentStatusWatcher.grafanaToken
self.grafanaURL = config.AgentStatusWatcher.grafanaURL
self.grafanaAPIName = config.AgentStatusWatcher.grafanaSSB
self.grafana = Grafana(_token,
configDict={"endpoint": self.grafanaURL})
# set pending percentages from config
self.pendingSlotsSitePercent = config.AgentStatusWatcher.pendingSlotsSitePercent
self.pendingSlotsTaskPercent = config.AgentStatusWatcher.pendingSlotsTaskPercent
self.runningExpressPercent = config.AgentStatusWatcher.runningExpressPercent
self.runningRepackPercent = config.AgentStatusWatcher.runningRepackPercent
# sites forced to down
self.forceSiteDown = getattr(config.AgentStatusWatcher,
'forceSiteDown', [])
# agent team (for dynamic threshold) and queueParams (drain mode)
self.teamName = config.Agent.teamName
self.agentsNumByTeam = getattr(config.AgentStatusWatcher,
'defaultAgentsNumByTeam', 5)
# only SSB sites
self.onlySSB = config.AgentStatusWatcher.onlySSB
# tier mode
self.tier0Mode = hasattr(config, "Tier0Feeder")
self.t1SitesCores = config.AgentStatusWatcher.t1SitesCores
# switch this component on/off
self.enabled = getattr(config.AgentStatusWatcher, 'enabled', True)
# set resource control
self.resourceControl = ResourceControl(config=self.config)
# wmstats connection
self.centralCouchDBReader = WMStatsReader(
self.config.General.centralWMStatsURL)
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config
self.accountantWorkSize = getattr(self.config.JobAccountant, 'accountantWorkSize', 100)
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "JobAccountant")
return
def __init__(self, queue):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.previousWorkList = []
def __init__(self, queue):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.previousWorkList = []
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.enabled = config.RucioInjector.enabled
# dataset rule creation has a larger polling cycle
self.pollRules = config.RucioInjector.pollIntervalRules
self.lastRulesExecTime = 0
self.createBlockRules = config.RucioInjector.createBlockRules
self.containerDiskRuleParams = config.RucioInjector.containerDiskRuleParams
self.containerDiskRuleRSEExpr = config.RucioInjector.containerDiskRuleRSEExpr
self.skipRulesForTiers = config.RucioInjector.skipRulesForTiers
self.listTiersToInject = config.RucioInjector.listTiersToInject
if config.RucioInjector.metaDIDProject not in RUCIO_VALID_PROJECT:
msg = "Component configured with an invalid 'project' DID: %s"
raise RucioInjectorException(msg % config.RucioInjector.metaDIDProject)
self.metaDIDProject = dict(project=config.RucioInjector.metaDIDProject)
# setup cache for container and blocks (containers can be much longer, make 6 days now)
self.containersCache = MemoryCache(config.RucioInjector.cacheExpiration * 3, set())
self.blocksCache = MemoryCache(config.RucioInjector.cacheExpiration, set())
self.scope = getattr(config.RucioInjector, "scope", "cms")
self.rucioAcct = config.RucioInjector.rucioAccount
self.rucio = Rucio(acct=self.rucioAcct,
hostUrl=config.RucioInjector.rucioUrl,
authUrl=config.RucioInjector.rucioAuthUrl,
configDict={'logger': self.logger})
# metadata dictionary information to be added to block/container rules
# cannot be a python dictionary, but a JSON string instead
self.metaData = json.dumps(dict(agentHost=config.Agent.hostName,
userAgent=config.Agent.agentName))
self.testRSEs = config.RucioInjector.RSEPostfix
self.filesToRecover = []
# output data placement has a different behaviour between T0 and Production agents
if hasattr(config, "Tier0Feeder"):
logging.info("RucioInjector running on a T0 WMAgent")
self.isT0agent = True
else:
self.isT0agent = False
if not self.listTiersToInject:
logging.info("Component configured to inject all the data tiers")
else:
logging.info("Component configured to only inject data for data tiers: %s",
self.listTiersToInject)
logging.info("Component configured to skip container rule creation for data tiers: %s",
self.skipRulesForTiers)
logging.info("Component configured to create block rules: %s", self.createBlockRules)
def __init__(self, config):
"""
___init___
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.dbsUrl = config.DBSInterface.globalDBSUrl
self.pollCounter = 0
self.subFrequency = None
if getattr(config.PhEDExInjector, "subscribeDatasets", False):
pollInterval = config.PhEDExInjector.pollInterval
subInterval = config.PhEDExInjector.subscribeInterval
self.subFrequency = max(1, int(round(subInterval/pollInterval)))
logging.info("SubscribeDataset and deleteBlocks will run every %d polling cycles", self.subFrequency)
# subscribe on first cycle
self.pollCounter = self.subFrequency - 1
# retrieving the node mappings is fickle and can fail quite often
self.phedex = PhEDEx({"endpoint": config.PhEDExInjector.phedexurl}, "json")
try:
nodeMappings = self.phedex.getNodeMap()
except:
time.sleep(2)
try:
nodeMappings = self.phedex.getNodeMap()
except:
time.sleep(4)
nodeMappings = self.phedex.getNodeMap()
# This will be used to map SE names which are stored in the DBSBuffer to
# PhEDEx node names. The first key will be the "kind" which consists
# of one of the following: MSS, Disk, Buffer. The next key will be the
# SE name.
self.seMap = {}
self.nodeNames = []
for node in nodeMappings["phedex"]["node"]:
if node["kind"] not in self.seMap:
self.seMap[node["kind"]] = {}
logging.info("Adding mapping %s -> %s", node["se"], node["name"])
self.seMap[node["kind"]][node["se"]] = node["name"]
self.nodeNames.append(node["name"])
self.phedexNodes = {'MSS': [], 'Disk': []}
for node in nodeMappings["phedex"]["node"]:
if node["kind"] in ["MSS", "Disk"]:
self.phedexNodes[node["kind"]].append(node["name"])
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "PhEDExInjector")
self.blocksToRecover = []
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.logger = logging.getLogger()
self.config = config.AsyncTransferTest
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
# state lists which shouldn't be populated in wmbs. (To prevent creating work before WQE status updated)
self.abortedAndForceCompleteWorkflowCache = self.reqmgr2Svc.getAbortedAndForceCompleteRequestsFromMemoryCache(expire=120)
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.logger = logging.getLogger()
self.config = config.AsyncTransferTest
self.logger.setLevel(self.config.log_level)
self.logger.debug('Configuration loaded')
def __init__(self, config):
BaseWorkerThread.__init__(self)
self.config = config
self.accountantWorkSize = getattr(self.config.JobAccountant,
'accountantWorkSize', 100)
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName = "JobAccountant")
return
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
def __init__(self, queue, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.queue = queue
self.config = config
self.reqmgr2Svc = ReqMgr(self.config.TaskArchiver.ReqMgr2ServiceURL)
# state lists which shouldn't be populated in wmbs. (To prevent creating work before WQE status updated)
self.abortedAndForceCompleteWorkflowCache = self.reqmgr2Svc.getAbortedAndForceCompleteRequestsFromMemoryCache(
)
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(
**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", None)
self.useReqMgrForCompletionCheck = getattr(
self.config.TaskArchiver, 'useReqMgrForCompletionCheck', True)
if not self.useReqMgrForCompletionCheck:
#sets the local monitor summary couch db
self.requestLocalCouchDB = RequestDBWriter(
self.config.AnalyticsDataCollector.localT0RequestDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralCouchDBWriter = self.requestLocalCouchDB
else:
self.centralCouchDBWriter = RequestDBWriter(
self.config.AnalyticsDataCollector.centralRequestDBURL)
self.reqmgr2Svc = ReqMgr(
self.config.TaskArchiver.ReqMgr2ServiceURL)
#TODO: remove this when reqmgr2 replace reqmgr completely (reqmgr2Only)
self.reqmgrSvc = RequestManager(
{'endpoint': self.config.TaskArchiver.ReqMgrServiceURL})
#Load the cleanout state ID and save it
stateIDDAO = self.daoFactory(classname="Jobs.GetStateID")
self.stateID = stateIDDAO.execute("cleanout")
return
def __init__(self, config):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package = "WMCore.WMBS",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.config = config
self.jobCacheDir = self.config.JobCreator.jobCacheDir
if getattr(self.config.TaskArchiver, "useWorkQueue", False) != False:
# Get workqueue setup from config unless overridden
if hasattr(self.config.TaskArchiver, 'WorkQueueParams'):
self.workQueue = localQueue(**self.config.TaskArchiver.WorkQueueParams)
else:
from WMCore.WorkQueue.WorkQueueUtils import queueFromConfig
self.workQueue = queueFromConfig(self.config)
else:
self.workQueue = None
self.timeout = getattr(self.config.TaskArchiver, "timeOut", 0)
self.nOffenders = getattr(self.config.TaskArchiver, 'nOffenders', 3)
# Set up optional histograms
self.histogramKeys = getattr(self.config.TaskArchiver, "histogramKeys", [])
self.histogramBins = getattr(self.config.TaskArchiver, "histogramBins", 10)
self.histogramLimit = getattr(self.config.TaskArchiver, "histogramLimit", 5.0)
# Start a couch server for getting job info
# from the FWJRs for committal to archive
try:
self.dbname = getattr(self.config.TaskArchiver, 'workloadSummaryCouchURL',
self.config.JobStateMachine.couchDBName)
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self.summarydb = getattr(self.config.TaskArchiver, "summaryDBName", self.dbname)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
self.workdatabase = self.couchdb.connectDatabase(self.dbname)
logging.debug("Using url %s" % self.config.JobStateMachine.couchurl)
logging.debug("Writing to %s" % self.dbname)
self.requireCouch = getattr(self.config.TaskArchiver, 'requireCouch', False)
except Exception, ex:
msg = "Error in connecting to couch.\n"
msg += str(ex)
logging.error(msg)
self.jobsdatabase = None
self.fwjrdatabase = None
if getattr(self.config.TaskArchiver, 'requireCouch', False):
raise TaskArchiverPollerException(msg)
def __init__(self, queue, config, reqMgr = None):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.wq = queue
self.config = config
if reqMgr:
self.reqMgr = reqMgr
else:
self.reqMgr = WorkQueueReqMgrInterface(**self.config)
self.previousState = {}
def __init__(self, queue, config, reqMgr=None):
"""
Initialise class members
"""
BaseWorkerThread.__init__(self)
self.wq = queue
self.config = config
if reqMgr:
self.reqMgr = reqMgr
else:
self.reqMgr = WorkQueueReqMgrInterface(**self.config)
self.previousState = {}
def __init__(self):
"""
Get a callback to the test object used to access "thread doing stuff"
data
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.dummySetupCallback = myThread.dummySetupCallback
self.dummyAlgoCallback = myThread.dummyAlgoCallback
self.dummyTerminateCallback = myThread.dummyTerminateCallback
if 'workerThreadManager' in dir(myThread):
self.workerThreadManager = myThread.workerThreadManager
def __init__(self):
"""
Get a callback to the test object used to access "thread doing stuff"
data
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
self.dummySetupCallback = myThread.dummySetupCallback
self.dummyAlgoCallback = myThread.dummyAlgoCallback
self.dummyTerminateCallback = myThread.dummyTerminateCallback
if 'workerThreadManager' in dir(myThread):
self.workerThreadManager = myThread.workerThreadManager
def __init__(self, config):
"""
init jobCreator
"""
BaseWorkerThread.__init__(self)
myThread = threading.currentThread()
# DAO factory for WMBS objects
self.daoFactory = DAOFactory(package="WMCore.WMBS",
logger=logging,
dbinterface=myThread.dbi)
self.setBulkCache = self.daoFactory(classname="Jobs.SetCache")
self.countJobs = self.daoFactory(
classname="Jobs.GetNumberOfJobsPerWorkflow")
self.subscriptionList = self.daoFactory(
classname="Subscriptions.ListIncomplete")
self.setFWJRPath = self.daoFactory(classname="Jobs.SetFWJRPath")
# information
self.config = config
# Variables
self.defaultJobType = config.JobCreator.defaultJobType
self.limit = getattr(config.JobCreator, 'fileLoadLimit', 500)
self.agentNumber = int(getattr(config.Agent, 'agentNumber', 0))
self.agentName = getattr(config.Agent, 'hostName', '')
self.glideinLimits = getattr(config.JobCreator, 'GlideInRestriction',
None)
# initialize the alert framework (if available - config.Alert present)
# self.sendAlert will be then be available
self.initAlerts(compName="JobCreator")
try:
self.jobCacheDir = getattr(
config.JobCreator, 'jobCacheDir',
os.path.join(config.JobCreator.componentDir, 'jobCacheDir'))
self.check()
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while setting up jobCacheDir!\n"
msg += str(ex)
logging.error(msg)
self.sendAlert(6, msg=msg)
raise JobCreatorException(msg)
self.changeState = ChangeState(self.config)
return