def __init__(self, componentName, pollInterval=None, heartbeatTimeout=7200,
logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attributes.
Every worker has a different instance of this class, thus they can have
a different polling interval and a different heartbeat timeout.
Finally, check to see if a transaction object has been created.
If none exists, create one but leave the transaction closed.
"""
self.componentName = componentName
self.pollInterval = pollInterval
self.heartbeatTimeout = heartbeatTimeout or 7200
self.compId = os.getpid()
WMConnectionBase.__init__(self, daoPackage="WMCore.Agent.Database",
logger=logger, dbi=dbi)
self.insertComp = self.daofactory(classname="InsertComponent")
self.existWorker = self.daofactory(classname="ExistWorker")
self.insertWorker = self.daofactory(classname="InsertWorker")
self.updateWorker = self.daofactory(classname="UpdateWorker")
self.updateErrorWorker = self.daofactory(classname="UpdateWorkerError")
self.getHeartbeat = self.daofactory(classname="GetHeartbeatInfo")
self.getAllHeartbeat = self.daofactory(classname="GetAllHeartbeatInfo")
def __init__(self, options):
"""
__init__
Initialize the object
"""
self.configFilePath = options.configFilePath
self.inputDataFilePath = options.inputDataFilePath
self.filesPerBlock = options.filesPerBlock
self.timeToClose = options.timeToClose
self.setup()
WMConnectionBase.__init__(self, "WMCore.WMBS")
self.dbsFilesToCreate = []
self.datasetAlgoID = collections.deque(maxlen = 1000)
self.datasetAlgoPaths = collections.deque(maxlen = 1000)
self.dbsLocations = collections.deque(maxlen = 1000)
self.workflowIDs = collections.deque(maxlen = 1000)
self.workflowPaths = collections.deque(maxlen = 1000)
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.dbsCreateFiles = self.dbsDaoFactory(classname = "DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
self.dbsSetRunLumi = self.dbsDaoFactory(classname = "DBSBufferFiles.AddRunLumi")
self.dbsInsertWorkflow = self.dbsDaoFactory(classname = "InsertWorkflow")
return
def __init__(self, config, couchDbName=None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName == None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
try:
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" %
self.dbname,
size=250)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" %
self.dbname,
size=250)
self.jsumdatabase = self.couchdb.connectDatabase(getattr(
self.config.JobStateMachine, 'jobSummaryDBName'),
size=250)
except Exception, ex:
logging.error("Error connecting to couch: %s" % str(ex))
self.jobsdatabase = None
self.fwjrdatabase = None
self.jsumdatabase = None
def __init__(self, config, couchDbName = None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName == None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
self.jobsdatabase = None
self.fwjrdatabase = None
self.jsumdatabase = None
self.statsumdatabase = None
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self._connectDatabases()
try:
self.dashboardReporter = DashboardReporter(config)
except Exception as ex:
logging.error("Error setting up the \
- dashboard reporter: %s" % str(ex))
raise
self.getCouchDAO = self.daofactory("Jobs.GetCouchID")
self.setCouchDAO = self.daofactory("Jobs.SetCouchID")
self.incrementRetryDAO = self.daofactory("Jobs.IncrementRetry")
self.workflowTaskDAO = self.daofactory("Jobs.GetWorkflowTask")
self.jobTypeDAO = self.daofactory("Jobs.GetType")
self.updateLocationDAO = self.daofactory("Jobs.UpdateLocation")
self.maxUploadedInputFiles = getattr(self.config.JobStateMachine, 'maxFWJRInputFiles', 1000)
return
def __init__(self, config, couchDbName = None, couchurl = None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
self.designDoc = "HarvestingDatasets"
if couchDbName == None:
self.dbname = getattr(self.config.HarvestingScheduler, "couchDBName",
"dqm_default")
else:
self.dbname = couchDbName
if couchurl is not None:
self.couchurl = couchurl
elif getattr(self.config.HarvestingScheduler, "couchurl",
None) is not None:
self.couchurl = self.config.HarvestingScheduler.couchurl
else:
self.couchurl = self.config.JobStateMachine.couchurl
try:
self.couchdb = CouchServer(self.couchurl)
if self.dbname not in self.couchdb.listDatabases():
self.createDatabase()
self.database = self.couchdb.connectDatabase(self.dbname,
size=_LIMIT)
except Exception, ex:
logging.error("Error connecting to couch: %s" % str(ex))
self.database = None
def __init__(self,
path,
id=-1,
processingVer=None,
acquisitionEra=None,
validStatus=None,
globalTag=None,
parent=None,
prep_id=None):
"""
Initialize the stored attributes and
database connection.
"""
WMConnectionBase.__init__(self, daoPackage="WMComponent.DBS3Buffer")
# Fill out the attributes
self['path'] = path
self['id'] = id
self['processingVer'] = processingVer
self['acquisitionEra'] = acquisitionEra
self['validStatus'] = validStatus
self['globalTag'] = globalTag
self['parent'] = parent
self['prep_id'] = prep_id
self['subscriptions'] = []
def __init__(self, config = None):
WMConnectionBase.__init__(self, daoPackage = "WMCore.ResourceControl")
self.wmbsDAOFactory = DAOFactory(package = "WMCore.WMBS",
logger = self.logger,
dbinterface = self.dbi)
self.config = config
return
def __init__(self, config=None):
WMConnectionBase.__init__(self, daoPackage="WMCore.ResourceControl")
self.wmbsDAOFactory = DAOFactory(package="WMCore.WMBS",
logger=self.logger,
dbinterface=self.dbi)
self.config = config
return
def __init__(self,
wmSpec,
taskName,
blockName=None,
mask=None,
cachepath='.'):
"""
_init_
Initialize DAOs and other things needed.
"""
self.block = blockName
self.mask = mask
self.wmSpec = wmSpec
self.topLevelTask = wmSpec.getTask(taskName)
self.cachepath = cachepath
self.isDBS = True
self.topLevelFileset = None
self.topLevelSubscription = None
self.topLevelTaskDBSBufferId = None
self.mergeOutputMapping = {}
# Initiate the pieces you need to run your own DAOs
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
# DAOs from WMBS for file commit
self.setParentage = self.daofactory(classname="Files.SetParentage")
self.setFileLocation = self.daofactory(
classname="Files.SetLocationForWorkQueue")
self.setFileAddChecksum = self.daofactory(
classname="Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname="Files.Add")
self.addToFileset = self.daofactory(classname="Files.AddDupsToFileset")
self.getLocations = self.daofactory(classname="Locations.ListSites")
self.getLocationInfo = self.daofactory(
classname="Locations.GetSiteInfo")
# DAOs from DBSBuffer
self.dbsCreateFiles = self.dbsDaoFactory(
classname="DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(
classname="DBSBufferFiles.AddChecksumByLFN")
self.dbsInsertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow")
# Added for file creation bookkeeping
self.dbsFilesToCreate = set()
self.wmbsFilesToCreate = set()
self.insertedBogusDataset = -1
return
def __init__(self, config, couchDbName=None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName is None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
self.jobsdatabase = None
self.fwjrdatabase = None
self.jsumdatabase = None
self.statsumdatabase = None
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self._connectDatabases()
try:
self.dashboardReporter = DashboardReporter(config)
except Exception as ex:
logging.error("Error setting up the dashboard reporter: %s", str(ex))
raise
self.getCouchDAO = self.daofactory("Jobs.GetCouchID")
self.setCouchDAO = self.daofactory("Jobs.SetCouchID")
self.incrementRetryDAO = self.daofactory("Jobs.IncrementRetry")
self.workflowTaskDAO = self.daofactory("Jobs.GetWorkflowTask")
self.jobTypeDAO = self.daofactory("Jobs.GetType")
self.updateLocationDAO = self.daofactory("Jobs.UpdateLocation")
self.getWorkflowSpecDAO = self.daofactory("Workflow.GetSpecAndNameFromTask")
self.maxUploadedInputFiles = getattr(self.config.JobStateMachine, 'maxFWJRInputFiles', 1000)
self.workloadCache = {}
return
def __init__(self):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for WMCore.WMBS as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
WMConnectionBase.__init__(self, daoPackage="WMCore.WMBS")
def __init__(self):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for WMCore.WMBS as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
WMConnectionBase.__init__(self, daoPackage = "WMCore.WMBS")
def __init__(self, config, insertStates=False):
"""
__init__
BossAir should work with the standard config
structure of WMAgent
"""
WMConnectionBase.__init__(self, daoPackage="WMCore.BossAir")
myThread = threading.currentThread()
self.config = config
self.plugins = {}
self.states = []
self.jobs = []
self.pluginDir = config.BossAir.pluginDir
# This is the default state jobs are created in
self.newState = getattr(config.BossAir, 'newState', 'New')
# Get any proxy info
self.checkProxy = getattr(config.BossAir, 'checkProxy', False)
self.cert = getattr(config.BossAir, 'cert', None)
self.stateMachine = ChangeState(self.config)
# Create a factory to load plugins
self.pluginFactory = WMFactory("plugins", self.pluginDir)
self.daoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.deleteDAO = self.daoFactory(classname="DeleteJobs")
self.stateDAO = self.daoFactory(classname="NewState")
self.loadByWMBSDAO = self.daoFactory(classname="LoadByWMBSID")
self.updateDAO = self.daoFactory(classname="UpdateJobs")
self.newJobDAO = self.daoFactory(classname="NewJobs")
self.runningJobDAO = self.daoFactory(classname="LoadRunning")
self.completeJobDAO = self.daoFactory(classname="LoadComplete")
self.loadJobsDAO = self.daoFactory(classname="LoadByStatus")
self.completeDAO = self.daoFactory(classname="CompleteJob")
self.monitorDAO = self.daoFactory(classname="JobStatusForMonitoring")
self.states = None
self.loadPlugin(insertStates)
return
def __init__(self, componentName, logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for WMCore.WorkQueue as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
WMConnectionBase.__init__(self, daoPackage = "WMCore.Agent.Database",
logger = logger, dbi = dbi)
self.componentName = componentName
self.pid = os.getpid()
def __init__(self, wmSpec, taskName, blockName = None, mask = None, cachepath = '.'):
"""
_init_
Initialize DAOs and other things needed.
"""
self.block = blockName
self.mask = mask
self.wmSpec = wmSpec
self.topLevelTask = wmSpec.getTask(taskName)
self.cachepath = cachepath
self.isDBS = True
self.topLevelFileset = None
self.topLevelSubscription = None
self.topLevelTaskDBSBufferId = None
self.mergeOutputMapping = {}
# Initiate the pieces you need to run your own DAOs
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
# DAOs from WMBS for file commit
self.setParentage = self.daofactory(classname = "Files.SetParentage")
self.setFileRunLumi = self.daofactory(classname = "Files.AddRunLumi")
self.setFileLocation = self.daofactory(classname = "Files.SetLocationForWorkQueue")
self.setFileAddChecksum = self.daofactory(classname = "Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname = "Files.Add")
self.addToFileset = self.daofactory(classname = "Files.AddDupsToFileset")
self.getLocations = self.daofactory(classname = "Locations.ListSites")
self.getLocationInfo = self.daofactory(classname = "Locations.GetSiteInfo")
# DAOs from DBSBuffer
self.dbsCreateFiles = self.dbsDaoFactory(classname = "DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
self.dbsInsertWorkflow = self.dbsDaoFactory(classname = "InsertWorkflow")
# Added for file creation bookkeeping
self.dbsFilesToCreate = []
self.addedLocations = []
self.wmbsFilesToCreate = []
self.insertedBogusDataset = -1
return
def __init__(self, config, insertStates=False):
"""
__init__
BossAir should work with the standard config
structure of WMAgent
"""
WMConnectionBase.__init__(self, daoPackage="WMCore.BossAir")
myThread = threading.currentThread()
self.config = config
self.plugins = {}
self.states = []
self.jobs = []
self.pluginDir = config.BossAir.pluginDir
# This is the default state jobs are created in
self.newState = getattr(config.BossAir, 'newState', 'New')
# Get any proxy info
self.checkProxy = getattr(config.BossAir, 'checkProxy', False)
self.cert = getattr(config.BossAir, 'cert', None)
self.stateMachine = ChangeState(self.config)
# Create a factory to load plugins
self.pluginFactory = WMFactory("plugins", self.pluginDir)
self.daoFactory = DAOFactory(package="WMCore.BossAir",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.deleteDAO = self.daoFactory(classname="DeleteJobs")
self.stateDAO = self.daoFactory(classname="NewState")
self.loadByWMBSDAO = self.daoFactory(classname="LoadByWMBSID")
self.updateDAO = self.daoFactory(classname="UpdateJobs")
self.newJobDAO = self.daoFactory(classname="NewJobs")
self.runningJobDAO = self.daoFactory(classname="LoadRunning")
self.completeJobDAO = self.daoFactory(classname="LoadComplete")
self.loadJobsDAO = self.daoFactory(classname="LoadByStatus")
self.completeDAO = self.daoFactory(classname="CompleteJob")
self.monitorDAO = self.daoFactory(classname="JobStatusForMonitoring")
self.states = None
self.loadPlugin(insertStates)
return
def __init__(self, logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for WMCore.WorkQueue as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
# only load dbi connection if we need it
if dbi or 'dbi' in dir(threading.currentThread()):
from WMCore.WMConnectionBase import WMConnectionBase
self.conn = WMConnectionBase(daoPackage="WMCore",
logger=logger,
dbi=dbi)
self.logger = self.conn.logger
else:
self.conn = None
if logger:
self.logger = logger
elif 'logger' in dir(threading.currentThread()):
self.logger = threading.currentThread().logger
else:
import logging
self.logger = logging.getLogger()
def __init__(self, componentName, logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for WMCore.WorkQueue as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
WMConnectionBase.__init__(self,
daoPackage="WMCore.Agent.Database",
logger=logger,
dbi=dbi)
self.componentName = componentName
self.pid = os.getpid()
def __init__(self, config, couchDbName = None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName == None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
try:
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self.jobsdatabase = self.couchdb.connectDatabase("%s/jobs" % self.dbname)
self.fwjrdatabase = self.couchdb.connectDatabase("%s/fwjrs" % self.dbname)
except Exception, ex:
logging.error("Error connecting to couch: %s" % str(ex))
self.jobsdatabase = None
self.fwjrdatabase = None
def __init__(self, config, couchDbName = None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName == None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self._connectDatabases()
try:
self.dashboardReporter = DashboardReporter(config)
except Exception, ex:
logging.error("Error setting up the \
- dashboard reporter: %s" % str(ex))
raise
def __init__(self, config, couchDbName=None):
WMObject.__init__(self, config)
WMConnectionBase.__init__(self, "WMCore.WMBS")
if couchDbName == None:
self.dbname = getattr(self.config.JobStateMachine, "couchDBName")
else:
self.dbname = couchDbName
self.couchdb = CouchServer(self.config.JobStateMachine.couchurl)
self._connectDatabases()
try:
self.dashboardReporter = DashboardReporter(config)
except Exception, ex:
logging.error("Error setting up the \
- dashboard reporter: %s" % str(ex))
raise
def __init__(self, path, id = -1,
processingVer = None, acquisitionEra = None, validStatus = None,
globalTag = None, parent = None):
"""
Initialize the stored attributes and
database connection.
"""
WMConnectionBase.__init__(self, daoPackage = "WMComponent.DBS3Buffer")
# Fill out the attributes
self['path'] = path
self['id'] = id
self['processingVer'] = processingVer
self['acquisitionEra'] = acquisitionEra
self['validStatus'] = validStatus
self['globalTag'] = globalTag
self['parent'] = parent
self['subscriptions'] = []
def __init__(self, componentName, pollInterval=None, logger=None, dbi=None):
"""
___init___
Initialize all the database connection attributes and the logging
attritbutes. Create a DAO factory for WMCore.WorkQueue as well. Finally,
check to see if a transaction object has been created. If none exists,
create one but leave the transaction closed.
"""
WMConnectionBase.__init__(self, daoPackage = "WMCore.Agent.Database",
logger = logger, dbi = dbi)
self.insertComp = self.daofactory(classname = "InsertComponent")
self.existWorker = self.daofactory(classname = "ExistWorker")
self.insertWorker = self.daofactory(classname="InsertWorker")
self.updateWorker = self.daofactory(classname="UpdateWorker")
self.updateErrorWorker = self.daofactory(classname = "UpdateWorkerError")
self.getHeartbeat = self.daofactory(classname = "GetHeartbeatInfo")
self.getAllHeartbeat = self.daofactory(classname = "GetAllHeartbeatInfo")
self.componentName = componentName
self.pid = os.getpid()
self.pollInterval = pollInterval
def __init__(self, config):
"""
__init__
Create all DAO objects that are used by this class.
"""
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.getOutputMapAction = self.daofactory(classname = "Jobs.GetOutputMap")
self.bulkAddToFilesetAction = self.daofactory(classname = "Fileset.BulkAddByLFN")
self.bulkParentageAction = self.daofactory(classname = "Files.AddBulkParentage")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
self.getParentInfoAction = self.daofactory(classname = "Files.GetParentInfo")
self.setParentageByJob = self.daofactory(classname = "Files.SetParentageByJob")
self.setFileRunLumi = self.daofactory(classname = "Files.AddRunLumi")
self.setFileLocation = self.daofactory(classname = "Files.SetLocationByLFN")
self.setFileAddChecksum = self.daofactory(classname = "Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname = "Files.Add")
self.jobCompleteInput = self.daofactory(classname = "Jobs.CompleteInput")
self.setBulkOutcome = self.daofactory(classname = "Jobs.SetOutcomeBulk")
self.getWorkflowSpec = self.daofactory(classname = "Workflow.GetSpecAndNameFromTask")
self.dbsStatusAction = self.dbsDaoFactory(classname = "DBSBufferFiles.SetStatus")
self.dbsParentStatusAction = self.dbsDaoFactory(classname = "DBSBufferFiles.GetParentStatus")
self.dbsChildrenAction = self.dbsDaoFactory(classname = "DBSBufferFiles.GetChildren")
self.dbsCreateFiles = self.dbsDaoFactory(classname = "DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
self.dbsSetRunLumi = self.dbsDaoFactory(classname = "DBSBufferFiles.AddRunLumi")
self.insertWorkflow = self.dbsDaoFactory(classname = "InsertWorkflow")
self.dbsNewAlgoAction = self.dbsDaoFactory(classname = "NewAlgo")
self.dbsNewDatasetAction = self.dbsDaoFactory(classname = "NewDataset")
self.dbsAssocAction = self.dbsDaoFactory(classname = "AlgoDatasetAssoc")
self.dbsExistsAction = self.dbsDaoFactory(classname = "DBSBufferFiles.ExistsForAccountant")
self.dbsLFNHeritage = self.dbsDaoFactory(classname = "DBSBufferFiles.BulkHeritageParent")
self.dbsSetDatasetAlgoAction = self.dbsDaoFactory(classname = "SetDatasetAlgo")
self.stateChanger = ChangeState(config)
# Decide whether or not to attach jobReport to returned value
self.returnJobReport = getattr(config.JobAccountant, 'returnReportFromWorker', False)
# Store location for the specs for DBS
self.specDir = getattr(config.JobAccountant, 'specDir', None)
# Hold data for later commital
self.dbsFilesToCreate = []
self.wmbsFilesToBuild = []
self.fileLocation = None
self.mergedOutputFiles = []
self.listOfJobsToSave = []
self.listOfJobsToFail = []
self.filesetAssoc = []
self.count = 0
self.datasetAlgoID = collections.deque(maxlen = 1000)
self.datasetAlgoPaths = collections.deque(maxlen = 1000)
self.dbsLocations = collections.deque(maxlen = 1000)
self.workflowIDs = collections.deque(maxlen = 1000)
self.workflowPaths = collections.deque(maxlen = 1000)
self.phedex = PhEDEx()
self.locLists = self.phedex.getNodeMap()
return
def __init__(self, config):
"""
__init__
Create all DAO objects that are used by this class.
"""
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getOutputMapAction = self.daofactory(
classname="Jobs.GetOutputMap")
self.bulkAddToFilesetAction = self.daofactory(
classname="Fileset.BulkAddByLFN")
self.bulkParentageAction = self.daofactory(
classname="Files.AddBulkParentage")
self.getJobTypeAction = self.daofactory(classname="Jobs.GetType")
self.getParentInfoAction = self.daofactory(
classname="Files.GetParentInfo")
self.setParentageByJob = self.daofactory(
classname="Files.SetParentageByJob")
self.setParentageByMergeJob = self.daofactory(
classname="Files.SetParentageByMergeJob")
self.setFileRunLumi = self.daofactory(classname="Files.AddRunLumi")
self.setFileLocation = self.daofactory(
classname="Files.SetLocationByLFN")
self.setFileAddChecksum = self.daofactory(
classname="Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname="Files.Add")
self.jobCompleteInput = self.daofactory(classname="Jobs.CompleteInput")
self.setBulkOutcome = self.daofactory(classname="Jobs.SetOutcomeBulk")
self.getWorkflowSpec = self.daofactory(
classname="Workflow.GetSpecAndNameFromTask")
self.getJobInfoByID = self.daofactory(classname="Jobs.LoadFromID")
self.getFullJobInfo = self.daofactory(
classname="Jobs.LoadForErrorHandler")
self.getJobTaskNameAction = self.daofactory(
classname="Jobs.GetFWJRTaskName")
self.pnn_to_psn = self.daofactory(
classname="Locations.GetPNNtoPSNMapping").execute()
self.dbsStatusAction = self.dbsDaoFactory(
classname="DBSBufferFiles.SetStatus")
self.dbsParentStatusAction = self.dbsDaoFactory(
classname="DBSBufferFiles.GetParentStatus")
self.dbsChildrenAction = self.dbsDaoFactory(
classname="DBSBufferFiles.GetChildren")
self.dbsCreateFiles = self.dbsDaoFactory(
classname="DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(
classname="DBSBufferFiles.AddChecksumByLFN")
self.dbsSetRunLumi = self.dbsDaoFactory(
classname="DBSBufferFiles.AddRunLumi")
self.dbsGetWorkflow = self.dbsDaoFactory(classname="ListWorkflow")
self.dbsLFNHeritage = self.dbsDaoFactory(
classname="DBSBufferFiles.BulkHeritageParent")
self.stateChanger = ChangeState(config)
# Decide whether or not to attach jobReport to returned value
self.returnJobReport = getattr(config.JobAccountant,
'returnReportFromWorker', False)
# Store location for the specs for DBS
self.specDir = getattr(config.JobAccountant, 'specDir', None)
# maximum RAW EDM size for Repack output before data is put into Error dataset and skips PromptReco
self.maxAllowedRepackOutputSize = getattr(
config.JobAccountant, 'maxAllowedRepackOutputSize',
12 * 1024 * 1024 * 1024)
# ACDC service
self.dataCollection = DataCollectionService(
url=config.ACDC.couchurl, database=config.ACDC.database)
jobDBurl = sanitizeURL(config.JobStateMachine.couchurl)['url']
jobDBName = config.JobStateMachine.couchDBName
jobCouchdb = CouchServer(jobDBurl)
self.fwjrCouchDB = jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
self.localWMStats = WMStatsWriter(config.TaskArchiver.localWMStatsURL,
appName="WMStatsAgent")
# Hold data for later commital
self.dbsFilesToCreate = []
self.wmbsFilesToBuild = []
self.wmbsMergeFilesToBuild = []
self.fileLocation = None
self.mergedOutputFiles = []
self.listOfJobsToSave = []
self.listOfJobsToFail = []
self.filesetAssoc = []
self.parentageBinds = []
self.parentageBindsForMerge = []
self.jobsWithSkippedFiles = {}
self.count = 0
self.datasetAlgoID = collections.deque(maxlen=1000)
self.datasetAlgoPaths = collections.deque(maxlen=1000)
self.dbsLocations = set()
self.workflowIDs = collections.deque(maxlen=1000)
self.workflowPaths = collections.deque(maxlen=1000)
self.phedex = PhEDEx()
self.locLists = self.phedex.getNodeMap()
return
def __init__(self, config):
"""
__init__
Create all DAO objects that are used by this class.
"""
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.getOutputMapAction = self.daofactory(
classname="Jobs.GetOutputMap")
self.bulkAddToFilesetAction = self.daofactory(
classname="Fileset.BulkAddByLFN")
self.bulkParentageAction = self.daofactory(
classname="Files.AddBulkParentage")
self.getJobTypeAction = self.daofactory(classname="Jobs.GetType")
self.getParentInfoAction = self.daofactory(
classname="Files.GetParentInfo")
self.setParentageByJob = self.daofactory(
classname="Files.SetParentageByJob")
self.setFileRunLumi = self.daofactory(classname="Files.AddRunLumi")
self.setFileLocation = self.daofactory(
classname="Files.SetLocationByLFN")
self.setFileAddChecksum = self.daofactory(
classname="Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname="Files.Add")
self.jobCompleteInput = self.daofactory(classname="Jobs.CompleteInput")
self.setBulkOutcome = self.daofactory(classname="Jobs.SetOutcomeBulk")
self.getWorkflowSpec = self.daofactory(
classname="Workflow.GetSpecAndNameFromTask")
self.dbsStatusAction = self.dbsDaoFactory(
classname="DBSBufferFiles.SetStatus")
self.dbsParentStatusAction = self.dbsDaoFactory(
classname="DBSBufferFiles.GetParentStatus")
self.dbsChildrenAction = self.dbsDaoFactory(
classname="DBSBufferFiles.GetChildren")
self.dbsCreateFiles = self.dbsDaoFactory(
classname="DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(
classname="DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(
classname="DBSBufferFiles.AddChecksumByLFN")
self.dbsSetRunLumi = self.dbsDaoFactory(
classname="DBSBufferFiles.AddRunLumi")
self.insertWorkflow = self.dbsDaoFactory(classname="InsertWorkflow")
self.dbsNewAlgoAction = self.dbsDaoFactory(classname="NewAlgo")
self.dbsNewDatasetAction = self.dbsDaoFactory(classname="NewDataset")
self.dbsAssocAction = self.dbsDaoFactory(classname="AlgoDatasetAssoc")
self.dbsExistsAction = self.dbsDaoFactory(
classname="DBSBufferFiles.ExistsForAccountant")
self.dbsLFNHeritage = self.dbsDaoFactory(
classname="DBSBufferFiles.BulkHeritageParent")
self.dbsSetDatasetAlgoAction = self.dbsDaoFactory(
classname="SetDatasetAlgo")
self.stateChanger = ChangeState(config)
# Decide whether or not to attach jobReport to returned value
self.returnJobReport = getattr(config.JobAccountant,
'returnReportFromWorker', False)
# Store location for the specs for DBS
self.specDir = getattr(config.JobAccountant, 'specDir', None)
# Hold data for later commital
self.dbsFilesToCreate = []
self.wmbsFilesToBuild = []
self.fileLocation = None
self.mergedOutputFiles = []
self.listOfJobsToSave = []
self.listOfJobsToFail = []
self.filesetAssoc = []
self.count = 0
self.datasetAlgoID = collections.deque(maxlen=1000)
self.datasetAlgoPaths = collections.deque(maxlen=1000)
self.dbsLocations = collections.deque(maxlen=1000)
self.workflowIDs = collections.deque(maxlen=1000)
self.workflowPaths = collections.deque(maxlen=1000)
self.phedex = PhEDEx()
self.locLists = self.phedex.getNodeMap()
return
def __init__(self, config):
"""
__init__
Create all DAO objects that are used by this class.
"""
WMConnectionBase.__init__(self, "WMCore.WMBS")
myThread = threading.currentThread()
self.dbsDaoFactory = DAOFactory(package = "WMComponent.DBS3Buffer",
logger = myThread.logger,
dbinterface = myThread.dbi)
self.getOutputMapAction = self.daofactory(classname = "Jobs.GetOutputMap")
self.bulkAddToFilesetAction = self.daofactory(classname = "Fileset.BulkAddByLFN")
self.bulkParentageAction = self.daofactory(classname = "Files.AddBulkParentage")
self.getJobTypeAction = self.daofactory(classname = "Jobs.GetType")
self.getParentInfoAction = self.daofactory(classname = "Files.GetParentInfo")
self.setParentageByJob = self.daofactory(classname = "Files.SetParentageByJob")
self.setFileRunLumi = self.daofactory(classname = "Files.AddRunLumi")
self.setFileLocation = self.daofactory(classname = "Files.SetLocationByLFN")
self.setFileAddChecksum = self.daofactory(classname = "Files.AddChecksumByLFN")
self.addFileAction = self.daofactory(classname = "Files.Add")
self.jobCompleteInput = self.daofactory(classname = "Jobs.CompleteInput")
self.setBulkOutcome = self.daofactory(classname = "Jobs.SetOutcomeBulk")
self.getWorkflowSpec = self.daofactory(classname = "Workflow.GetSpecAndNameFromTask")
self.getJobInfoByID = self.daofactory(classname = "Jobs.LoadFromID")
self.getFullJobInfo = self.daofactory(classname = "Jobs.LoadForErrorHandler")
self.dbsStatusAction = self.dbsDaoFactory(classname = "DBSBufferFiles.SetStatus")
self.dbsParentStatusAction = self.dbsDaoFactory(classname = "DBSBufferFiles.GetParentStatus")
self.dbsChildrenAction = self.dbsDaoFactory(classname = "DBSBufferFiles.GetChildren")
self.dbsCreateFiles = self.dbsDaoFactory(classname = "DBSBufferFiles.Add")
self.dbsSetLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.SetLocationByLFN")
self.dbsInsertLocation = self.dbsDaoFactory(classname = "DBSBufferFiles.AddLocation")
self.dbsSetChecksum = self.dbsDaoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
self.dbsSetRunLumi = self.dbsDaoFactory(classname = "DBSBufferFiles.AddRunLumi")
self.dbsGetWorkflow = self.dbsDaoFactory(classname = "ListWorkflow")
self.dbsLFNHeritage = self.dbsDaoFactory(classname = "DBSBufferFiles.BulkHeritageParent")
self.stateChanger = ChangeState(config)
# Decide whether or not to attach jobReport to returned value
self.returnJobReport = getattr(config.JobAccountant, 'returnReportFromWorker', False)
# Store location for the specs for DBS
self.specDir = getattr(config.JobAccountant, 'specDir', None)
# ACDC service
self.dataCollection = DataCollectionService(url = config.ACDC.couchurl,
database = config.ACDC.database)
jobDBurl = sanitizeURL(config.JobStateMachine.couchurl)['url']
jobDBName = config.JobStateMachine.couchDBName
jobCouchdb = CouchServer(jobDBurl)
self.fwjrCouchDB = jobCouchdb.connectDatabase("%s/fwjrs" % jobDBName)
self.localWMStats = WMStatsWriter(config.TaskArchiver.localWMStatsURL)
# Hold data for later commital
self.dbsFilesToCreate = []
self.wmbsFilesToBuild = []
self.fileLocation = None
self.mergedOutputFiles = []
self.listOfJobsToSave = []
self.listOfJobsToFail = []
self.filesetAssoc = []
self.parentageBinds = []
self.jobsWithSkippedFiles = {}
self.count = 0
self.datasetAlgoID = collections.deque(maxlen = 1000)
self.datasetAlgoPaths = collections.deque(maxlen = 1000)
self.dbsLocations = collections.deque(maxlen = 1000)
self.workflowIDs = collections.deque(maxlen = 1000)
self.workflowPaths = collections.deque(maxlen = 1000)
self.phedex = PhEDEx()
self.locLists = self.phedex.getNodeMap()
return
