def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError("You must set the DATABASE environment variable to run tests")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print("****WARNING: the DBHOST environment variable will be deprecated soon***")
print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
return config
def __init__(self, configFile):
self.config = loadConfigurationFile(configFile)
# current dir when writing stub files.
self.currentDir = None
# parsed component information from configuration file.
self.components = {}
# parsed synchronizers (trigger) information from configuration file.
self.synchronizers = {}
# default stub message that is put in every generated file.
self.stubmsg = """
def __init__(self, configFile):
self.config = loadConfigurationFile(configFile)
# current dir when writing stub files.
self.currentDir = None
# parsed component information from configuration file.
self.components = {}
# parsed synchronizers (trigger) information from configuration file.
self.synchronizers = {}
# default stub message that is put in every generated file.
self.stubmsg = """
def testE(self):
"""test save/load """
testValues = [
"string", 123, 123.456,
["list", 789, 10.1 ],
{ "dict1" : "value", "dict2" : 10.0 }
]
config = Configuration()
for x in range(0, 5):
config.section_("Section%s" % x)
config.component_("Component%s" % x)
sect = getattr(config, "Section%s" % x)
comp = getattr(config, "Component%s" % x)
sect.document_("This is Section%s" % x)
comp.document_("This is Component%s" % x)
for i in range(0, 5):
setattr(comp, "Parameter%s" % i, testValues[i])
setattr(sect, "Parameter%s" % i, testValues[i])
comp.document_("This is Parameter%s" % i,
"Parameter%s" %i)
sect.document_("This is Parameter%s" %i,
"Parameter%s" %i)
stringSave = str(config)
documentSave = config.documentedString_()
commentSave = config.commentedString_()
saveConfigurationFile(config, self.normalSave)
saveConfigurationFile(config, self.docSave, document = True)
saveConfigurationFile(config, self.commentSave, comment = True)
plainConfig = loadConfigurationFile(self.normalSave)
docConfig = loadConfigurationFile(self.docSave)
commentConfig = loadConfigurationFile(self.commentSave)
def upload(self):
"""
Mimics creation of component and handles come messages.
Upload just 10 files for testing ONLY
"""
#return True
# read the default config first.
config = loadConfigurationFile(os.path.join(os.getenv('WMCOREBASE'), \
'src/python/WMComponent/DBSUpload/DefaultConfig.py'))
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "DBS"
config.Agent.agentName = "DBS Upload"
config.section_("General")
config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
config.CoreDatabase.dialect = 'mysql'
#config.CoreDatabase.socket = os.getenv("DBSOCK")
config.CoreDatabase.user = os.getenv("DBUSER")
config.CoreDatabase.passwd = os.getenv("DBPASS")
config.CoreDatabase.hostname = os.getenv("DBHOST")
config.CoreDatabase.name = os.getenv("DBNAME")
testDBSUpload = DBSUpload(config)
testDBSUpload.prepareToStart()
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# for testing purposes we use this method instead of the
# StartComponent one.
testDBSUpload.handleMessage('BufferSuccess', \
'NoPayLoad')
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
DBSBufferInterface._teardown = True
def bufferFWJR(self, fwjr):
"""
Mimics creation of component and handles JobSuccess messages.
"""
# read the default config first.
config = loadConfigurationFile(os.path.join(os.getenv('WMCOREBASE'), \
'src/python/WMComponent/DBSBuffer/DefaultConfig.py'))
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "DBS"
config.Agent.agentName = "DBS Buffer"
config.section_("General")
config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
config.CoreDatabase.dialect = 'mysql'
config.CoreDatabase.user = os.getenv("DBUSER")
config.CoreDatabase.passwd = os.getenv("DBPASS")
config.CoreDatabase.hostname = os.getenv("DBHOST")
config.CoreDatabase.name = os.getenv("DBNAME")
myThread = threading.currentThread()
myThread.logger = logging.getLogger('DBSBufferInterface')
myThread.dialect = 'MySQL'
options = {}
options['unix_socket'] = os.getenv("DBSOCK")
dbFactory = DBFactory(myThread.logger, os.getenv("DATABASE"), \
options)
testDBSBuffer = DBSBuffer(config)
testDBSBuffer.prepareToStart()
myThread.dbi = dbFactory.connect()
myThread.transaction = Transaction(myThread.dbi)
testDBSBuffer.handleMessage('JobSuccess', fwjr)
while threading.activeCount() > 1:
print('Currently: '+str(threading.activeCount())+\
' Threads. Wait until all our threads have finished')
time.sleep(1)
DBSBufferInterface._teardown = True
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") )
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print "****WARNING: the DBHOST environment variable will be deprecated soon***"
print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
# after this you can augment it with whatever you need.
return config
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if (os.getenv('DATABASE') == None):
raise RuntimeError, \
"You must set the DATABASE environment variable to run tests"
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL( os.getenv("DATABASE") )
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print "****WARNING: the DBHOST environment variable will be deprecated soon***"
print "****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
# after this you can augment it with whatever you need.
return config
def initComponents(self, configPath):
"""
_initComponents_
Start up the various components using the config
"""
if os.path.isfile(configPath):
# Read the config
config = loadConfigurationFile(configPath)
else:
msg = "No config file at desired location"
logging.error(msg)
raise Exception(msg)
masterConfig = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
config.CoreDatabase.socket = masterConfig.CoreDatabase.socket
config.CoreDatabase.connectUrl = masterConfig.CoreDatabase.connectUrl
# Have to do this because the agent hard codes its dirs
oldWorkDir = config.General.workDir
for compName in (config.listComponents_() + config.listWebapps_()):
component = getattr(config, compName)
for var in component.listSections_():
value = getattr(component, var)
if type(value) == str:
if re.search(oldWorkDir, value):
# Then set it
setattr(component, var,
value.replace(oldWorkDir, self.testDir))
elif type(value) == list:
# Go through it one component at a time
for element in value:
if type(element) == str and re.search(
oldWorkDir, element):
index = value.index(element)
value.remove(element)
value.insert(
index, value.replace(oldWorkDir, self.testDir))
setattr(component, var, value)
elif type(value) == dict:
for key in value.keys():
if type(value[key]) == str and re.search(
oldWorkDir, value[key]):
value[key] = value[key].replace(
oldWorkDir, self.testDir)
setattr(component, var, value)
compList = (config.listComponents_() + config.listWebapps_())
components = []
config.JobStateMachine.couchurl = os.environ['COUCHURL']
config.JobStateMachine.couchDBName = self.dbName
config.ACDC.couchurl = os.environ['COUCHURL']
config.ACDC.database = '%s/acdc' % self.dbName
config.TaskArchiver.workloadSummaryCouchDBName = '%s/workloadsummary' % self.dbName
config.TaskArchiver.workloadSummaryCouchURL = os.environ['COUCHURL']
if hasattr(config, 'WorkQueueManager'):
config.WorkQueueManager.couchurl = os.environ['COUCHURL']
config.WorkQueueManager.dbname = '%s/workqueue' % self.dbName
if hasattr(config, 'WorkloadSummary'):
config.WorkloadSummary.couchurl = os.environ['COUCHURL']
config.WorkloadSummary.database = '%s/workloadsummary' % self.dbName
# Get all components
components.append(JobCreator(config=config))
components.append(JobSubmitter(config=config))
components.append(JobTracker(config=config))
components.append(JobAccountant(config=config))
components.append(JobArchiver(config=config))
components.append(TaskArchiver(config=config))
components.append(ErrorHandler(config=config))
components.append(RetryManager(config=config))
components.append(DBSUpload(config=config))
# Now the optional ones
if 'PhEDExInjector' in compList:
components.append(PhEDExInjector(config=config))
# Init threads:
for component in components:
component.initInThread()
# preInitialize
for component in components:
component.preInitialization()
for component in components:
component.prepareToStop()
return
import time
import inspect
import threading
#from MessageService.MessageService import MessageService
from WMCore.Agent.Configuration import loadConfigurationFile
from WMCore.WMFactory import WMFactory
from WMCore.Database.Transaction import Transaction
from WMCore.Database.DBFactory import DBFactory
#for logging
import logging
#TaskQueue
from PilotManager.PilotManagerComponent import PilotManagerComponent
config = loadConfigurationFile(\
'/data/khawar/prototype/PilotManager/DefaultConfig.py')
config.section_("General")
config.General.workDir = '/data/khawar/prototype/work/PilotManager'
config.Agent.componentName='PilotManager'
config.section_("CoreDatabase")
config.CoreDatabase.dialect = 'mysql'
config.CoreDatabase.socket = '/data/khawar/PAProd/0_12_13/prodAgent/mysqldata/mysql.sock'
config.CoreDatabase.user = '******'
config.CoreDatabase.passwd = '98passwd'
config.CoreDatabase.hostname = 'localhost'
config.CoreDatabase.name = 'ProdAgentDB'
def initComponents(self, configPath):
"""
_initComponents_
Start up the various components using the config
"""
if os.path.isfile(configPath):
# Read the config
config = loadConfigurationFile(configPath)
else:
msg = "No config file at desired location"
logging.error(msg)
raise Exception(msg)
masterConfig = self.testInit.getConfiguration()
config.Agent.useHeartbeat = False
config.CoreDatabase.socket = masterConfig.CoreDatabase.socket
config.CoreDatabase.connectUrl = masterConfig.CoreDatabase.connectUrl
# Have to do this because the agent hard codes its dirs
oldWorkDir = config.General.workDir
for compName in (config.listComponents_() + config.listWebapps_()):
component = getattr(config, compName)
for var in component.listSections_():
value = getattr(component, var)
if type(value) == str:
if re.search(oldWorkDir, value):
# Then set it
setattr(component, var, value.replace(oldWorkDir, self.testDir))
elif type(value) == list:
# Go through it one component at a time
for element in value:
if type(element) == str and re.search(oldWorkDir, element):
index = value.index(element)
value.remove(element)
value.insert(index, value.replace(oldWorkDir, self.testDir))
setattr(component, var, value)
elif type(value) == dict:
for key in value.keys():
if type(value[key]) == str and re.search(oldWorkDir, value[key]):
value[key] = value[key].replace(oldWorkDir, self.testDir)
setattr(component, var, value)
compList = (config.listComponents_() + config.listWebapps_())
components = []
config.JobStateMachine.couchurl = os.environ['COUCHURL']
config.JobStateMachine.couchDBName = self.dbName
config.ACDC.couchurl = os.environ['COUCHURL']
config.ACDC.database = '%s/acdc' % self.dbName
config.TaskArchiver.workloadSummaryCouchDBName = '%s/workloadsummary' % self.dbName
config.TaskArchiver.workloadSummaryCouchURL = os.environ['COUCHURL']
if hasattr(config, 'WorkQueueManager'):
config.WorkQueueManager.couchurl = os.environ['COUCHURL']
config.WorkQueueManager.dbname = '%s/workqueue' % self.dbName
if hasattr(config, 'WorkloadSummary'):
config.WorkloadSummary.couchurl = os.environ['COUCHURL']
config.WorkloadSummary.database = '%s/workloadsummary' % self.dbName
# Get all components
components.append(JobCreator(config = config))
components.append(JobSubmitter(config = config))
components.append(JobTracker(config = config))
components.append(JobAccountant(config = config))
components.append(JobArchiver(config = config))
components.append(TaskArchiver(config = config))
components.append(ErrorHandler(config = config))
components.append(RetryManager(config = config))
components.append(DBSUpload(config = config))
# Now the optional ones
if 'PhEDExInjector' in compList:
components.append(PhEDExInjector(config = config))
# Init threads:
for component in components:
component.initInThread()
# preInitialize
for component in components:
component.preInitialization()
for component in components:
component.prepareToStop()
return
if __name__ == '__main__':
if len(sys.argv) < 2:
usage()
sys.exit(1)
datasetPath = sys.argv[1]
if len(sys.argv) < 3: #No config path sent along
configPath = os.getenv('WMAGENT_CONFIG', None)
else:
configPath = sys.argv[2]
if not os.path.isfile(configPath):
configPath = os.getenv('WMAGENT_CONFIG', None)
if configPath:
config = loadConfigurationFile(configPath)
else:
print "Error! No config could be found"
sys.exit(2)
wmInit = WMInit()
wmInit.setLogging()
wmInit.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
migrateFileBlocks = MigrateFileBlocks(config = config)
migrateFileBlocks.migrateDataset(datasetPath = datasetPath)
from PilotManager.PilotManagerComponent import PilotManagerComponent
# //
# // Find and load the Configuration
#//
try:
config = loadProdAgentConfiguration()
compCfg = config.getConfig("PilotManager")
except StandardError, ex:
msg = "Error reading configuration:\n"
msg += str(ex)
raise RuntimeError, msg
compCfg['ComponentDir'] = os.path.expandvars(compCfg['ComponentDir'])
config = loadConfigurationFile(compCfg['defaultConfig'])
config.PilotManagerComponent.componentDir=compCfg['ComponentDir']
if ( compCfg.has_key("plugin") ):
config.PilotManagerComponent.plugin=compCfg['plugin']
if ( compCfg.has_key("tarPath") ):
config.PilotManagerComponent.tarPath=compCfg['tarPath']
else:
config.PilotManagerComponent.tarPath=compCfg['ComponentDir']
config.PilotManagerComponent.pilotCode=compCfg['pilotCode']
config.PilotManagerComponent.tqAddress=compCfg['tqAddress']
#settig up the configuration for PilotMonitor
config.section_("CoreDatabase")
if __name__ == '__main__':
if len(sys.argv) < 2:
usage()
sys.exit(1)
datasetPath = sys.argv[1]
if len(sys.argv) < 3: #No config path sent along
configPath = os.getenv('WMAGENT_CONFIG', None)
else:
configPath = sys.argv[2]
if not os.path.isfile(configPath):
configPath = os.getenv('WMAGENT_CONFIG', None)
if configPath:
config = loadConfigurationFile(configPath)
else:
print "Error! No config could be found"
sys.exit(2)
wmInit = WMInit()
wmInit.setLogging()
wmInit.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
migrateFileBlocks = MigrateFileBlocks(config=config)
migrateFileBlocks.migrateDataset(datasetPath=datasetPath)
import threading
# from MessageService.MessageService import MessageService
from WMCore.Agent.Configuration import loadConfigurationFile
from WMCore.WMFactory import WMFactory
from WMCore.Database.Transaction import Transaction
from WMCore.Database.DBFactory import DBFactory
# for logging
import logging
# TaskQueue
from PilotMonitor.PilotMonitorComponent import PilotMonitorComponent
config = loadConfigurationFile("/data/khawar/prototype/PilotMonitor/DefaultConfig.py")
config.section_("General")
config.General.workDir = "/data/khawar/prototype/work/PilotMonitor"
config.Agent.componentName = "PilotMonitor"
config.section_("CoreDatabase")
config.CoreDatabase.dialect = "mysql"
config.CoreDatabase.socket = "/data/khawar/PAProd/0_12_13/prodAgent/mysqldata/mysql.sock"
config.CoreDatabase.user = "******"
config.CoreDatabase.passwd = "98passwd"
config.CoreDatabase.hostname = "localhost"
config.CoreDatabase.name = "ProdAgentDB"
