def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname = "Locations.New")
getFileLoc = daofactory(classname = "Files.GetLocation")
fileInFileset = daofactory(classname = "Files.InFileset")
logging.debug("DBSFeeder is processing %s" % \
filesetToProcess.name)
logging.debug("the filesetBase name %s" \
% (filesetToProcess.name).split(":")[0])
LASTIME = filesetToProcess.lastUpdate
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
# get list of files
tries = 1
while True:
try:
blocks = self.dbsReader.getFiles(\
(filesetToProcess.name).split(":")[0])
now = time.time()
logging.debug("DBS queries done ...")
break
except DBSReaderError, ex:
logging.error("DBS error: %s, cannot get files for %s" % \
(str(ex), filesetToProcess.name))
# Close fileset
filesetToProcess.markOpen(False)
return
# connection error, retry
except DbsConnectionError, ex:
logging.error("Unable to connect to DBS, retrying: " + \
str(ex))
if tries > self.connectionAttempts: #too many errors - bail out
return
tries = tries + 1
def connectToDB():
"""
_connectToDB_
Connect to the database specified in the WMAgent config.
"""
if "WMAGENT_CONFIG" not in os.environ:
print(
"Please set WMAGENT_CONFIG to point at your WMAgent configuration."
)
sys.exit(1)
if not os.path.exists(os.environ["WMAGENT_CONFIG"]):
print("Can't find config: %s" % os.environ["WMAGENT_CONFIG"])
sys.exit(1)
wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
if not hasattr(wmAgentConfig, "CoreDatabase"):
print("Your config is missing the CoreDatabase section.")
socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
(dialect, junk) = connectUrl.split(":", 1)
myWMInit = WMInit()
myWMInit.setDatabaseConnection(dbConfig=connectUrl,
dialect=dialect,
socketLoc=socketLoc)
return
def main():
myPhedex = PhEDEx()
config = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
config.CoreDatabase.dialect = "mysql"
init = WMInit()
init.setDatabaseConnection(config.CoreDatabase.connectUrl, config.CoreDatabase.dialect, config.CoreDatabase.socket)
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMComponent.PhEDExInjector.Database", logger=logging, dbinterface=myThread.dbi)
getUninjectedDAO = daofactory(classname="GetUninjectedFiles")
uninjectedFiles = getUninjectedDAO.execute()
for location in uninjectedFiles:
for dataset in uninjectedFiles[location]:
for block in uninjectedFiles[location][dataset]:
result = myPhedex.getReplicaInfoForFiles(dataset=dataset, block=block)
phedexBlock = result["phedex"]["block"]
if not phedexBlock:
continue
phedexBlock = phedexBlock[0]
filesInjected = [x["name"] for x in phedexBlock["file"]]
for fileInfo in uninjectedFiles[location][dataset][block]["files"]:
lfn = fileInfo["lfn"]
if lfn in filesInjected:
print lfn
return 0
def main():
config = loadConfigurationFile(os.environ['WMAGENT_CONFIG'])
config.CoreDatabase.dialect = 'oracle'
init = WMInit()
init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect)
couchDB = Database('wmagent_jobdump/fwjrs', '')
couchDB2 = Database('wmagent_jobdump/jobs', '')
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS",
logger = logging,
dbinterface = myThread.dbi)
getJobsDAO = daofactory(classname = "Jobs.GetAllJobs")
completedJobs = getJobsDAO.execute(state = 'complete')
candidates = []
while len(completedJobs):
candidates = []
chunk = completedJobs[:500]
completedJobs = completedJobs[500:]
result = couchDB.loadView('FWJRDump', 'outputByJobID', keys = chunk)
rows = result['rows']
for entry in rows:
candidates.append(entry['key'])
for jobId in candidates:
doc = couchDB2.document(str(jobId))
last = max(map(int, doc['states'].keys()))
lastState = doc['states'][str(last)]['newstate']
if lastState == 'success':
print jobId
def connectToDB():
"""
_connectToDB_
Connect to the database specified in the WMAgent config.
"""
if not os.environ.has_key("WMAGENT_CONFIG"):
print "Please set WMAGENT_CONFIG to point at your WMAgent configuration."
sys.exit(1)
if not os.path.exists(os.environ["WMAGENT_CONFIG"]):
print "Can't find config: %s" % os.environ["WMAGENT_CONFIG"]
sys.exit(1)
wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
if not hasattr(wmAgentConfig, "CoreDatabase"):
print "Your config is missing the CoreDatabase section."
socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
(dialect, junk) = connectUrl.split(":", 1)
myWMInit = WMInit()
myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect,
socketLoc = socketLoc)
return
def __init__(self, testClassName="Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __init__(self, testClassName = "Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("DBSFeeder is processing %s" % \
filesetToProcess.name)
logging.debug("the filesetBase name %s" \
% (filesetToProcess.name).split(":")[0])
LASTIME = filesetToProcess.lastUpdate
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
# get list of files
tries = 1
while True:
try:
blocks = self.dbsReader.getFiles(\
(filesetToProcess.name).split(":")[0])
now = time.time()
logging.debug("DBS queries done ...")
break
except DBSReaderError, ex:
logging.error("DBS error: %s, cannot get files for %s" % \
(str(ex), filesetToProcess.name))
# Close fileset
filesetToProcess.markOpen(False)
return
# connection error, retry
except DbsConnectionError, ex:
logging.error("Unable to connect to DBS, retrying: " + \
str(ex))
if tries > self.connectionAttempts: #too many errors - bail out
return
tries = tries + 1
def testMySQLDatabase(self):
"""
Testing MySQL basic operations
"""
dialect = os.environ.get("DIALECT", "MySQL")
if dialect.lower() == 'oracle':
# this test can only run for MySQL
return
init = WMInit()
url = os.environ.get("DATABASE")
sock = os.environ.get("DBSOCK", None)
init.setDatabaseConnection(url, dialect, sock)
selectDbName = "SELECT DATABASE() AS dbname"
destroyDbName = "DROP DATABASE %s"
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear non-existant DB should work
init.clearDatabase()
init.setSchema(modules=['WMCore.WMBS'])
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
self.assertEqual(myThread.dbi.engine.name, "mysql")
self.assertTrue(
myThread.dbi.engine.url.database in ("wmcore_unittest",
"WMCore_unit_test"))
self.assertEqual(myThread.dbi.engine.url.get_backend_name(),
"mysql")
self.assertEqual(myThread.dbi.engine.url.get_driver_name(),
"mysqldb")
self.assertEqual(myThread.dbi.engine.url.host, "localhost")
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.dbi.processData(destroyDbName % dbName)
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, None)
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, a)
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.transaction.begin()
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
else:
init.clearDatabase()
return
class TestInit(object):
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName="Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if self.testDir != None:
try:
shutil.rmtree(self.testDir)
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel=logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName,
self.testClassName,
logExists=False,
logLevel=logLevel)
def generateWorkDir(self,
config=None,
deleteOnDestruction=True,
setTmpDir=False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'oracle':
return 'Oracle'
elif dialectPart == 'http':
return 'CouchDB'
else:
raise RuntimeError("Unrecognized dialect %s" % dialectPart)
def setDatabaseConnection(self,
connectUrl=None,
socket=None,
destroyAllDatabase=False):
"""
Set up the database connection by retrieving the environment
parameters.
The destroyAllDatabase option is for testing ONLY. Never flip that switch
on in any other instance where you don't know what you're doing.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases or destroyAllDatabase:
self.clearDatabase()
# Have to check whether or not database is empty
# If the database is not empty when we go to set the schema, abort!
try:
result = self.init.checkDatabaseContents()
except Exception as e:
logging.debug(
"Error checking DB contents, assume DB does not exist")
logging.debug(str(e))
return
if len(result) > 0:
msg = "Database not empty, cannot set schema !\n"
msg += str(result)
logging.error(msg)
raise TestInitException(msg)
return
def setSchema(self, customModules=[], useDefault=True, params=None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter for unique modules
modules = []
for module in (defaultModules + customModules):
if module not in modules:
modules.append(module)
try:
self.init.setSchema(modules, params=params)
except Exception as ex:
print(traceback.format_exc())
raise ex
# store the list of modules we've added to the DB
# again filter for unique modules
modules = []
for module in (defaultModules + customModules + self.currModules):
if module not in modules:
modules.append(module)
self.currModules = modules
return
def getDBInterface(self):
"shouldbe called after connection is made"
if not self.hasDatabase:
return
myThread = threading.currentThread()
return myThread.dbi
def getConfiguration(self,
configurationFile=None,
connectUrl=None,
socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.General.ReqMgr2ServiceURL = "http://localhost/reqmgr2"
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError(
"You must set the DATABASE environment variable to run tests"
)
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(
os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print(
"****WARNING: the DBHOST environment variable will be deprecated soon***"
)
print(
"****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****"
)
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (
config.JobStateMachine.couchurl,
config.JobStateMachine.jobSummaryDBName)
return config
def clearDatabase(self, modules=[]):
"""
Database deletion. Global, ignore modules.
"""
if not self.hasDatabase:
return
self.init.clearDatabase()
return
def attemptToCloseDBConnections(self):
return
myThread = threading.currentThread()
print("Closing DB")
try:
if not myThread.transaction \
and not myThread.transaction.conn \
and not myThread.transaction.conn.closed:
myThread.transaction.conn.close()
myThread.transaction.conn = None
print("Connection Closed")
except Exception as e:
print("tried to close DBI but failed: %s" % e)
try:
if hasattr(myThread, "dbFactory"):
del myThread.dbFactory
print("dbFactory removed")
except Exception as e:
print("tried to delete factory but failed %s" % e)
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
LASTIME = filesetToProcess.lastUpdate
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((
filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = ((filesetToProcess.name\
).split(":")[0]).split('/')[3]
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0 queries done ...")
now = time.time()
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
try:
locationNew.execute(siteName="caf.cern.ch",
seName="caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
for run in files['runs']:
runSet = set()
runSet.add(Run(run, *files['runs'][run]))
newfile.addRunSet(runSet)
else:
newfile.loadData()
fileLoc = getFileLoc.execute(file=files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
# else:
# logging.debug("File already associated to %s" %fileLoc)
LOCK.release()
if len(newfile["runs"]):
val = 0
for run in newfile['runs']:
if run.run < int(startRun):
val = 1
break
if not val:
listFile = fileInFileset.execute\
(filesetToProcess.id)
if {'fileid': newfile['id']} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
logging.debug(
"new file created/loaded added by T0AST..."
)
except Exception as e:
logging.debug("Error when adding new files in T0AST...")
logging.debug(e)
logging.debug(format_exc())
LOCK.release()
filesetToProcess.commit()
else:
logging.debug("nothing to do in T0AST...")
# For reopned fileset or empty
# try until the purge time is reached
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0AST ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
class TestInit(object):
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName = "Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if self.testDir != None:
try:
shutil.rmtree( self.testDir )
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel = logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName, self.testClassName,
logExists = False, logLevel = logLevel)
def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'oracle':
return 'Oracle'
elif dialectPart == 'http':
return 'CouchDB'
else:
raise RuntimeError("Unrecognized dialect %s" % dialectPart)
def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
"""
Set up the database connection by retrieving the environment
parameters.
The destroyAllDatabase option is for testing ONLY. Never flip that switch
on in any other instance where you don't know what you're doing.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases or destroyAllDatabase:
self.clearDatabase()
# Have to check whether or not database is empty
# If the database is not empty when we go to set the schema, abort!
try:
result = self.init.checkDatabaseContents()
except OperationalError:
logging.debug("Error checking DB contents, assume DB does not exist")
return
if len(result) > 0:
msg = "Database not empty, cannot set schema !\n"
msg += str(result)
logging.error(msg)
raise TestInitException(msg)
return
def setSchema(self, customModules = [], useDefault = True, params = None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter out unique modules
modules = {}
for module in (defaultModules + customModules):
modules[module] = 'done'
try:
self.init.setSchema(modules.keys(), params = params)
except Exception as ex:
print(traceback.format_exc())
raise ex
# store the list of modules we've added to the DB
modules = {}
for module in (defaultModules + customModules + self.currModules):
modules[module] = 'done'
self.currModules = modules.keys()
return
def getDBInterface(self):
"shouldbe called after connection is made"
if not self.hasDatabase:
return
myThread = threading.currentThread()
return myThread.dbi
def getConfiguration(self, configurationFile = None, connectUrl = None, socket=None):
"""
Loads (if available) your configuration file and augments
it with the standard settings used in multiple tests.
"""
if configurationFile != None:
config = loadConfigurationFile(configurationFile)
else:
config = Configuration()
# some general settings that would come from the general default
# config file
config.Agent.contact = "*****@*****.**"
config.Agent.teamName = "Lakers"
config.Agent.agentName = "Lebron James"
config.Agent.hostName = "testhost.laker.world"
config.section_("General")
# If you need a testDir, call testInit.generateWorkDir
# config.General.workDir = os.getenv("TESTDIR")
config.section_("CoreDatabase")
if connectUrl:
config.CoreDatabase.connectUrl = connectUrl
config.CoreDatabase.dialect = self.getBackendFromDbURL(connectUrl)
config.CoreDatabase.socket = socket or os.getenv("DBSOCK")
else:
if os.getenv('DATABASE') == None:
raise RuntimeError("You must set the DATABASE environment variable to run tests")
config.CoreDatabase.connectUrl = os.getenv("DATABASE")
config.CoreDatabase.dialect = self.getBackendFromDbURL(os.getenv("DATABASE"))
config.CoreDatabase.socket = os.getenv("DBSOCK")
if os.getenv("DBHOST"):
print("****WARNING: the DBHOST environment variable will be deprecated soon***")
print("****WARNING: UPDATE YOUR ENVIRONMENT OR TESTS WILL FAIL****")
# after this you can augment it with whatever you need.
couchurl = os.getenv("COUCHURL")
config.section_("ACDC")
config.ACDC.couchurl = couchurl
config.ACDC.database = "wmagent_acdc_t"
config.component_("JobStateMachine")
config.JobStateMachine.couchurl = couchurl
config.JobStateMachine.couchDBName = "wmagent_job_test"
config.JobStateMachine.jobSummaryDBName = "job_summary"
config.JobStateMachine.summaryStatsDBName = "stat_summary_test"
config.component_("JobAccountant")
config.JobAccountant.pollInterval = 60
config.JobAccountant.componentDir = os.getcwd()
config.JobAccountant.logLevel = 'SQLDEBUG'
config.component_("TaskArchiver")
config.TaskArchiver.localWMStatsURL = "%s/%s" % (config.JobStateMachine.couchurl, config.JobStateMachine.jobSummaryDBName)
config.TaskArchiver.ReqMgrSeviceURL = "request manager service url"
config.TaskArchiver.ReqMgr2ServiceURL = "https://cmsweb-dev.cern.ch/reqmgr2"
return config
def clearDatabase(self, modules = []):
"""
Database deletion. Global, ignore modules.
"""
if not self.hasDatabase:
return
self.init.clearDatabase()
return
def attemptToCloseDBConnections(self):
return
myThread = threading.currentThread()
print("Closing DB")
try:
if not myThread.transaction \
and not myThread.transaction.conn \
and not myThread.transaction.conn.closed:
myThread.transaction.conn.close()
myThread.transaction.conn = None
print("Connection Closed")
except Exception as e:
print("tried to close DBI but failed: %s" % e)
try:
if hasattr(myThread, "dbFactory"):
del myThread.dbFactory
print("dbFactory removed")
except Exception as e:
print("tried to delete factory but failed %s" % e)
else:
stri=time.ctime().split()
stri1=stri[2]
stri2=stri[3].replace(":","")
arguments["ProcessingVersion"] = '%s_%s'%(stri1,stri2)
wmAgentConfig = loadConfigurationFile(os.environ["WMAGENT_CONFIG"])
if not hasattr(wmAgentConfig, "CoreDatabase"):
print "Your config is missing the CoreDatabase section."
socketLoc = getattr(wmAgentConfig.CoreDatabase, "socket", None)
connectUrl = getattr(wmAgentConfig.CoreDatabase, "connectUrl", None)
(dialect, junk) = connectUrl.split(":", 1)
myWMInit = WMInit()
myWMInit.setDatabaseConnection(dbConfig = connectUrl, dialect = dialect,
socketLoc = socketLoc)
workloadName = "CmsRunAnalysis-%s" % arguments["ProcessingVersion"]
workloadFile = "CmsRunAnalysis-%s.pkl" % arguments["ProcessingVersion"]
os.mkdir(workloadName)
cmsRunAna = AnalysisWorkloadFactory()
workload = cmsRunAna(workloadName, arguments)
taskMaker = TaskMaker(workload, os.path.join(os.getcwd(), workloadName))
taskMaker.skipSubscription = True
taskMaker.processWorkload()
workload.save(os.path.join(workloadName, workloadFile))
class TestInit:
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName="Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if (self.testDir != None):
try:
shutil.rmtree(self.testDir)
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel=logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName,
self.testClassName,
logExists=False,
logLevel=logLevel)
def generateWorkDir(self,
config=None,
deleteOnDestruction=True,
setTmpDir=False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'sqlite':
return 'SQLite'
elif dialectPart == 'oracle':
return 'Oracle'
elif dialectPart == 'http':
return 'CouchDB'
else:
raise RuntimeError, "Unrecognized dialect %s" % dialectPart
def setDatabaseConnection(self,
connectUrl=None,
socket=None,
destroyAllDatabase=False):
"""
Set up the database connection by retrieving the environment
parameters.
The destroyAllDatabase option is for testing ONLY. Never flip that switch
on in any other instance where you don't know what you're doing.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases or destroyAllDatabase:
self.clearDatabase()
# Have to check whether or not database is empty
# If the database is not empty when we go to set the schema, abort!
result = self.init.checkDatabaseContents()
if len(result) > 0:
msg = "Database not empty, cannot set schema !\n"
msg += str(result)
logging.error(msg)
raise TestInitException(msg)
return
def setSchema(self, customModules=[], useDefault=True, params=None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter out unique modules
modules = {}
for module in (defaultModules + customModules):
modules[module] = 'done'
try:
self.init.setSchema(modules.keys(), params=params)
except Exception, ex:
print traceback.format_exc()
raise ex
# store the list of modules we've added to the DB
modules = {}
for module in (defaultModules + customModules + self.currModules):
modules[module] = 'done'
self.currModules = modules.keys()
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), os.getenv("DIALECT"), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
fileInFileset = daofactory(classname="Files.InFileset")
logging.debug("the T0Feeder is processing %s" % filesetToProcess.name)
logging.debug("the fileset name %s" % (filesetToProcess.name).split(":")[0])
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
LASTIME = filesetToProcess.lastUpdate
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split("/")[1]
processedDataset = ((filesetToProcess.name).split(":")[0]).split("/")[2]
dataTier = ((filesetToProcess.name).split(":")[0]).split("/")[3]
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % (
fileType,
LASTIME,
primaryDataset,
processedDataset,
dataTier,
)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(url + "/" + "?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0 queries done ...")
now = time.time()
LASTIME = int(newFilesList["end_time"]) + 1
break
# process all files
if len(newFilesList["files"]):
try:
locationNew.execute(siteName="caf.cern.ch", seName="caf.cern.ch")
except Exception, e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList["files"]:
# Assume parents aren't asked
newfile = File(str(files["lfn"]), size=files["file_size"], events=files["events"])
try:
LOCK.acquire()
if newfile.exists() == False:
newfile.create()
for run in files["runs"]:
runSet = set()
runSet.add(Run(run, *files["runs"][run]))
newfile.addRunSet(runSet)
else:
newfile.loadData()
fileLoc = getFileLoc.execute(file=files["lfn"])
if "caf.cern.ch" not in fileLoc:
newfile.setLocation("caf.cern.ch")
# else:
# logging.debug("File already associated to %s" %fileLoc)
LOCK.release()
if len(newfile["runs"]):
val = 0
for run in newfile["runs"]:
if run.run < int(startRun):
val = 1
break
if not val:
listFile = fileInFileset.execute(filesetToProcess.id)
if {"fileid": newfile["id"]} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(int(newFilesList["end_time"]) + 1)
filesetToProcess.commit()
logging.debug("new file created/loaded added by T0AST...")
except Exception, e:
logging.debug("Error when adding new files in T0AST...")
logging.debug(e)
logging.debug(format_exc())
LOCK.release()
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname = "Locations.New")
getFileLoc = daofactory(classname = "Files.GetLocation")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = (((filesetToProcess.name\
).split(":")[0]).split('/')[3]).split('-')[0]
# Fisrt call to T0 db for this fileset
# Here add test for the closed fileset
LASTIME = filesetToProcess.lastUpdate
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRun queries done ...")
now = time.time()
filesetToProcess.last_update = now
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
LOCK.acquire()
try:
locationNew.execute(siteName = "caf.cern.ch", seName = "caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug( format_exc() )
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
if newfile.exists() == False :
newfile.create()
else:
newfile.loadData()
#Add run test if already exist
for run in files['runs']:
if startRun != 'None' and int(startRun) <= int(run):
# ToDo: Distinguish between
# filestA-RunX and filesetA-Run[0-9]*
filesetRun = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split\
('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\
).split('/')[2])+'/'+((((filesetToProcess.name).split\
(':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\
(run)+":"+":".join((filesetToProcess.name).split(':')[1:] \
) )
if filesetRun.exists() == False :
filesetRun.create()
else:
filesetRun.loadData()
# Add test runs already there
# (for growing dataset) -
# to support file with different runs and lumi
if not newfile['runs']:
runSet = set()
runSet.add(Run( run, *files['runs'][run]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(file = files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
filesetRun.addFile(newfile)
logging.debug("new file created/loaded added by T0ASTRun...")
filesetRun.commit()
except Exception as e:
logging.debug("Error when adding new files in T0ASTRun...")
logging.debug(e)
logging.debug( format_exc() )
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
LOCK.release()
else:
logging.debug("nothing to do...")
# For re-opned fileset or empty, try until the purge time
if (int(now)/3600 - LASTIME/3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if LASTIME:
myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
for listRun in requestResult[0]:
if int(startRun) <= int(listRun['run']):
if listRun['status'] =='CloseOutExport' or \
listRun['status'] =='Complete' or listRun['status'] ==\
'CloseOutT1Skimming':
closeFileset = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
(((filesetToProcess.name).split(':')[0]).split('/')[1]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
[2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\
+":"+":".join((filesetToProcess.name).split(':')[1:] ) )
if closeFileset.exists() != False :
closeFileset = Fileset( id = closeFileset.exists())
closeFileset.loadData()
if closeFileset.open == True:
closeFileset.markOpen(False)
# Commit the fileset
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0ASTRun ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now)/3600 - LASTIME/3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
class TestInit:
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName = "Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if (self.testDir != None):
try:
shutil.rmtree( self.testDir )
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel = logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName, self.testClassName,
logExists = False, logLevel = logLevel)
def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'sqlite':
return 'SQLite'
elif dialectPart == 'oracle':
return 'Oracle'
else:
raise RuntimeError, "Unrecognized dialect %s" % dialectPart
def eraseEverythingInDatabase(self):
if not self.hasDatabase:
return
if trashDatabases:
dbi = self.getDBInterface()
dialect = self.coreConfig.CoreDatabase.dialect
formatter = DBFormatter(logging.getLogger(''), dbi)
if (dialect == 'MySQL'):
formatter.sql = r"SHOW TABLES"
result = formatter.execute()
formatter.sql = "SET foreign_key_checks = 0"
formatter.execute()
allTables = []
for oneTable in result:
allTables.append( oneTable[0] )
if len(allTables) == 0:
return
query = "DROP TABLE IF EXISTS `%s`" % ("`,`".join(allTables))
formatter.sql = query
formatter.execute()
formatter.sql = "SET foreign_key_checks = 1"
formatter.execute()
elif (dialect == 'SQLite'):
formatter.sql = "SELECT name FROM sqlite_master WHERE type='table' ORDER BY name;"
result = formatter.execute()
for oneTable in result:
# sqlite stores some magic in the database
if ( oneTable[0].startswith('sqlite_') ):
continue
query = "DROP TABLE IF EXISTS %s" % oneTable[0]
failCount = 0
for x in range(5):
try:
formatter.sql = query
formatter.execute()
except Exception:
# sleep a sec and try again
failCount = failCount + 1
if (failCount == 5):
raise
else:
print "Attempting to wait before clearing SQLite database again"
time.sleep(1)
elif (dialect == 'Oracle'):
print "Rookie, fix blowing away oracle in TestInit. thanks"
pass
else:
raise RuntimeError, "This dialect is unsupported by trashDatabases"
pass
else:
pass
def setDatabaseConnection(self, connectUrl=None, socket=None):
"""
Set up the database connection by retrieving the environment
parameters.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(
config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases:
# we are going to own ths database.
# ...but the code isn't ready yet
self.eraseEverythingInDatabase()
pass
def setSchema(self, customModules = [], useDefault = True, params = None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter out unique modules
modules = {}
for module in (defaultModules + customModules):
modules[module] = 'done'
try:
self.init.setSchema(modules.keys(), params = params)
except Exception, ex:
try:
self.clearDatabase(modules = modules.keys())
except:
pass
raise ex
# store the list of modules we've added to the DB
modules = {}
for module in (defaultModules + customModules + self.currModules):
modules[module] = 'done'
self.currModules = modules.keys()
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
lastFileset = daofactory(classname = "Fileset.ListFilesetByTask")
lastWorkflow = daofactory(classname = "Workflow.LoadFromTask")
subsRun = daofactory(\
classname = "Subscriptions.LoadFromFilesetWorkflow")
successJob = daofactory(classname = "Subscriptions.SucceededJobs")
allJob = daofactory(classname = "Subscriptions.Jobs")
fileInFileset = daofactory(classname = "Files.InFileset")
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
fileType = (filesetToProcess.name).split(":")[2]
crabTask = filesetToProcess.name.split(":")[0]
LASTIME = filesetToProcess.lastUpdate
tries = 1
while True:
try:
myRequester = JSONRequests(url = "vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRunChain feeder queries done ...")
now = time.time()
break
for listRun in requestResult[0]:
if startRun != 'None' and int(listRun['run']) >= int(startRun):
if listRun['status'] =='CloseOutExport' or listRun\
['status']=='Complete' or listRun['status']=='CloseOutT1Skimming':
crabWorkflow = lastWorkflow.execute(task=crabTask)
crabFileset = lastFileset.execute\
(task=crabTask)
crabrunFileset = Fileset(\
name = crabFileset[0]["name"].split(':')[0].split\
('-Run')[0]+ '-Run' + str(listRun['run']) + ":" + \
":".join(crabFileset[0]['name'].split(':')[1:]) )
if crabrunFileset.exists() > 0:
crabrunFileset.load()
currSubs = subsRun.execute\
(crabrunFileset.id, crabWorkflow[0]['id'])
if currSubs:
listsuccessJob = successJob.execute(\
subscription=currSubs['id'])
listallJob = allJob.execute(\
subscription=currSubs['id'])
if len(listsuccessJob) == len(listallJob):
for currid in listsuccessJob:
currjob = Job( id = currid )
currjob.load()
logging.debug("Reading FJR %s" %currjob['fwjr_path'])
jobReport = readJobReport(currjob['fwjr_path'])
if len(jobReport) > 0:
if jobReport[0].files:
for newFile in jobReport[0].files:
logging.debug(\
"Output path %s" %newFile['LFN'])
newFileToAdd = File(\
lfn=newFile['LFN'], locations ='caf.cern.ch')
LOCK.acquire()
if newFileToAdd.exists\
() == False :
newFileToAdd.create()
else:
newFileToAdd.loadData()
LOCK.release()
listFile = \
fileInFileset.execute(filesetToProcess.id)
if {'fileid': \
newFileToAdd['id']} not in listFile:
filesetToProcess.addFile(\
newFileToAdd)
filesetToProcess\
.setLastUpdate(now)
filesetToProcess.commit()
logging.debug(\
"new file created/loaded and added by T0ASTRunChain...")
elif jobReport[0].analysisFiles:
for newFile in jobReport\
[0].analysisFiles:
logging.debug(\
"Ouput path %s " %newFile['LFN'])
newFileToAdd = File(\
lfn=newFile['LFN'], locations ='caf.cern.ch')
LOCK.acquire()
if newFileToAdd.exists\
() == False :
newFileToAdd.create()
else:
newFileToAdd.loadData()
LOCK.release()
listFile = \
fileInFileset.execute(filesetToProcess.id)
if {'fileid': newFileToAdd\
['id']} not in listFile:
logging.debug\
("%s loaded and added by T0ASTRunChain" %newFile['LFN'])
filesetToProcess.addFile\
(newFileToAdd)
filesetToProcess.\
setLastUpdate(now)
filesetToProcess.commit()
logging.debug(\
"new file created/loaded added by T0ASTRunChain...")
else: break #Missed fjr - Try next time
# Commit the fileset
logging.debug("Test purge in T0ASTRunChain ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
# For re-opned fileset or empty, try until the purge time
if (int(now)/3600 - LASTIME/3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if (int(now)/3600 - LASTIME/3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
if __name__ == '__main__':
if len(sys.argv) < 2:
usage()
sys.exit(1)
datasetPath = sys.argv[1]
if len(sys.argv) < 3: #No config path sent along
configPath = os.getenv('WMAGENT_CONFIG', None)
else:
configPath = sys.argv[2]
if not os.path.isfile(configPath):
configPath = os.getenv('WMAGENT_CONFIG', None)
if configPath:
config = loadConfigurationFile(configPath)
else:
print "Error! No config could be found"
sys.exit(2)
wmInit = WMInit()
wmInit.setLogging()
wmInit.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
migrateFileBlocks = MigrateFileBlocks(config=config)
migrateFileBlocks.migrateDataset(datasetPath=datasetPath)
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
global LOCK
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname="Locations.New")
getFileLoc = daofactory(classname="Files.GetLocation")
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
startRun = (filesetToProcess.name).split(":")[3]
fileType = (filesetToProcess.name).split(":")[2]
# url builder
primaryDataset = ((filesetToProcess.name).split(":")[0]).split('/')[1]
processedDataset = ((
filesetToProcess.name).split(":")[0]).split('/')[2]
dataTier = (((filesetToProcess.name\
).split(":")[0]).split('/')[3]).split('-')[0]
# Fisrt call to T0 db for this fileset
# Here add test for the closed fileset
LASTIME = filesetToProcess.lastUpdate
url = "/tier0/listfilesoverinterval/%s/%s/%s/%s/%s" % \
(fileType, LASTIME, primaryDataset,processedDataset, dataTier)
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get(\
url+"/"+"?return_type=text/json%2Bdas")
newFilesList = requestResult[0]["results"]
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRun queries done ...")
now = time.time()
filesetToProcess.last_update = now
LASTIME = int(newFilesList['end_time']) + 1
break
# process all files
if len(newFilesList['files']):
LOCK.acquire()
try:
locationNew.execute(siteName="caf.cern.ch",
seName="caf.cern.ch")
except Exception as e:
logging.debug("Error when adding new location...")
logging.debug(e)
logging.debug(format_exc())
for files in newFilesList['files']:
# Assume parents aren't asked
newfile = File(str(files['lfn']), \
size = files['file_size'], events = files['events'])
try:
if newfile.exists() == False:
newfile.create()
else:
newfile.loadData()
#Add run test if already exist
for run in files['runs']:
if startRun != 'None' and int(startRun) <= int(run):
# ToDo: Distinguish between
# filestA-RunX and filesetA-Run[0-9]*
filesetRun = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split\
('/')[1])+'/'+(((filesetToProcess.name).split(':')[0]\
).split('/')[2])+'/'+((((filesetToProcess.name).split\
(':')[0]).split('/')[3]).split('-')[0])+'-'+'Run'+str\
(run)+":"+":".join((filesetToProcess.name).split(':')[1:] \
) )
if filesetRun.exists() == False:
filesetRun.create()
else:
filesetRun.loadData()
# Add test runs already there
# (for growing dataset) -
# to support file with different runs and lumi
if not newfile['runs']:
runSet = set()
runSet.add(Run(run, *files['runs'][run]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(file=files['lfn'])
if 'caf.cern.ch' not in fileLoc:
newfile.setLocation("caf.cern.ch")
filesetRun.addFile(newfile)
logging.debug(
"new file created/loaded added by T0ASTRun...")
filesetRun.commit()
except Exception as e:
logging.debug("Error when adding new files in T0ASTRun...")
logging.debug(e)
logging.debug(format_exc())
filesetToProcess.setLastUpdate\
(int(newFilesList['end_time']) + 1)
filesetToProcess.commit()
LOCK.release()
else:
logging.debug("nothing to do...")
# For re-opned fileset or empty, try until the purge time
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if LASTIME:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
for listRun in requestResult[0]:
if int(startRun) <= int(listRun['run']):
if listRun['status'] =='CloseOutExport' or \
listRun['status'] =='Complete' or listRun['status'] ==\
'CloseOutT1Skimming':
closeFileset = Fileset( name = (((\
filesetToProcess.name).split(':')[0]).split('/')[0])+'/'+\
(((filesetToProcess.name).split(':')[0]).split('/')[1]\
)+'/'+(((filesetToProcess.name).split(':')[0]).split('/')\
[2])+'/'+((((filesetToProcess.name).split(':')[0]).split\
('/')[3]).split('-')[0])+'-'+'Run'+str(listRun['run'])\
+":"+":".join((filesetToProcess.name).split(':')[1:] ) )
if closeFileset.exists() != False:
closeFileset = Fileset(id=closeFileset.exists())
closeFileset.loadData()
if closeFileset.open == True:
closeFileset.markOpen(False)
# Commit the fileset
filesetToProcess.commit()
# Commit the fileset
logging.debug("Test purge in T0ASTRun ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
def testB_Database(self):
"""
_Database_
Testing the database stuff.
"""
init = WMInit()
url = os.environ.get("DATABASE")
dialect = os.environ.get("DIALECT")
sock = os.environ.get("DBSOCK", None)
init.setDatabaseConnection(url, dialect, sock)
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear one after another should work
init.setSchema(modules = ['WMCore.WMBS'])
init.clearDatabase()
init.setSchema(modules = ['WMCore.WMBS'])
init.clearDatabase()
# Clear non-existant DB should work
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
myThread.dbi.processData("DROP DATABASE %s" % dbName)
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, None)
init.clearDatabase()
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
init.setSchema(modules = ['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData("SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.transaction.begin()
init.setSchema(modules = ['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
init.clearDatabase()
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
lastFileset = daofactory(classname="Fileset.ListFilesetByTask")
lastWorkflow = daofactory(classname="Workflow.LoadFromTask")
subsRun = daofactory(\
classname = "Subscriptions.LoadFromFilesetWorkflow")
successJob = daofactory(classname="Subscriptions.SucceededJobs")
allJob = daofactory(classname="Subscriptions.Jobs")
fileInFileset = daofactory(classname="Files.InFileset")
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
logging.debug("the T0Feeder is processing %s" % \
filesetToProcess.name)
logging.debug("the fileset name %s" % \
(filesetToProcess.name).split(":")[0])
fileType = (filesetToProcess.name).split(":")[2]
crabTask = filesetToProcess.name.split(":")[0]
LASTIME = filesetToProcess.lastUpdate
tries = 1
while True:
try:
myRequester = JSONRequests(url="vocms52.cern.ch:8889")
requestResult = myRequester.get("/tier0/runs")
except:
logging.debug("T0Reader call error...")
if tries == self.maxRetries:
return
else:
tries += 1
continue
logging.debug("T0ASTRunChain feeder queries done ...")
now = time.time()
break
for listRun in requestResult[0]:
if startRun != 'None' and int(listRun['run']) >= int(startRun):
if listRun['status'] =='CloseOutExport' or listRun\
['status']=='Complete' or listRun['status']=='CloseOutT1Skimming':
crabWorkflow = lastWorkflow.execute(task=crabTask)
crabFileset = lastFileset.execute\
(task=crabTask)
crabrunFileset = Fileset(\
name = crabFileset[0]["name"].split(':')[0].split\
('-Run')[0]+ '-Run' + str(listRun['run']) + ":" + \
":".join(crabFileset[0]['name'].split(':')[1:]) )
if crabrunFileset.exists() > 0:
crabrunFileset.load()
currSubs = subsRun.execute\
(crabrunFileset.id, crabWorkflow[0]['id'])
if currSubs:
listsuccessJob = successJob.execute(\
subscription=currSubs['id'])
listallJob = allJob.execute(\
subscription=currSubs['id'])
if len(listsuccessJob) == len(listallJob):
for currid in listsuccessJob:
currjob = Job(id=currid)
currjob.load()
logging.debug("Reading FJR %s" %
currjob['fwjr_path'])
jobReport = readJobReport(
currjob['fwjr_path'])
if len(jobReport) > 0:
if jobReport[0].files:
for newFile in jobReport[0].files:
logging.debug(\
"Output path %s" %newFile['LFN'])
newFileToAdd = File(\
lfn=newFile['LFN'], locations ='caf.cern.ch')
LOCK.acquire()
if newFileToAdd.exists\
() == False :
newFileToAdd.create()
else:
newFileToAdd.loadData()
LOCK.release()
listFile = \
fileInFileset.execute(filesetToProcess.id)
if {'fileid': \
newFileToAdd['id']} not in listFile:
filesetToProcess.addFile(\
newFileToAdd)
filesetToProcess\
.setLastUpdate(now)
filesetToProcess.commit()
logging.debug(\
"new file created/loaded and added by T0ASTRunChain...")
elif jobReport[0].analysisFiles:
for newFile in jobReport\
[0].analysisFiles:
logging.debug(\
"Ouput path %s " %newFile['LFN'])
newFileToAdd = File(\
lfn=newFile['LFN'], locations ='caf.cern.ch')
LOCK.acquire()
if newFileToAdd.exists\
() == False :
newFileToAdd.create()
else:
newFileToAdd.loadData()
LOCK.release()
listFile = \
fileInFileset.execute(filesetToProcess.id)
if {'fileid': newFileToAdd\
['id']} not in listFile:
logging.debug\
("%s loaded and added by T0ASTRunChain" %newFile['LFN'])
filesetToProcess.addFile\
(newFileToAdd)
filesetToProcess.\
setLastUpdate(now)
filesetToProcess.commit()
logging.debug(\
"new file created/loaded added by T0ASTRunChain...")
else:
break #Missed fjr - Try next time
# Commit the fileset
logging.debug("Test purge in T0ASTRunChain ...")
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
# For re-opned fileset or empty, try until the purge time
if (int(now) / 3600 - LASTIME / 3600) > self.reopenTime:
filesetToProcess.setLastUpdate(time.time())
filesetToProcess.commit()
if (int(now) / 3600 - LASTIME / 3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
def testOracleDatabase(self):
"""
Testing Oracle basic operations
"""
dialect = os.environ.get("DIALECT", "MySQL")
if dialect.lower() == 'mysql':
# this test can only run for Oracle
return
init = WMInit()
url = os.environ.get("DATABASE")
init.setDatabaseConnection(url, dialect)
selectDbName = "SELECT ora_database_name FROM DUAL"
destroyDb = """DECLARE
BEGIN
execute immediate 'purge recyclebin';
-- Tables
FOR o IN (SELECT table_name name FROM user_tables) LOOP
execute immediate 'drop table ' || o.name || ' cascade constraints';
END LOOP;
-- Sequences
FOR o IN (SELECT sequence_name name FROM user_sequences) LOOP
execute immediate 'drop sequence ' || o.name;
END LOOP;
-- Triggers
FOR o IN (SELECT trigger_name name FROM user_triggers) LOOP
execute immediate 'drop trigger ' || o.name;
END LOOP;
-- Synonyms
FOR o IN (SELECT synonym_name name FROM user_synonyms) LOOP
execute immediate 'drop synonym ' || o.name;
END LOOP;
-- Functions
FOR o IN (SELECT object_name name FROM user_objects WHERE object_type = 'FUNCTION') LOOP
execute immediate 'drop function ' || o.name;
END LOOP;
-- Procedures
FOR o IN (SELECT object_name name FROM user_objects WHERE object_type = 'PROCEDURE') LOOP
execute immediate 'drop procedure ' || o.name;
END LOOP;
execute immediate 'purge recyclebin';
END;"""
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear non-existant DB should work
init.clearDatabase()
init.setSchema(modules=['WMCore.WMBS'])
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
self.assertEqual(myThread.dbi.engine.name, "oracle")
self.assertIsNone(myThread.dbi.engine.url.database)
self.assertEqual(myThread.dbi.engine.url.get_backend_name(),
"oracle")
self.assertEqual(myThread.dbi.engine.url.get_driver_name(),
"cx_oracle")
self.assertEqual(myThread.dbi.engine.url.host, "INT2R_NOLB")
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertTrue(dbName)
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertTrue(dbName)
myThread.dbi.processData(destroyDb)
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(selectDbName)[0].fetchall()[0][0]
self.assertTrue(dbName)
myThread.transaction.begin()
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
else:
init.clearDatabase()
if __name__ == '__main__':
if len(sys.argv) < 2:
usage()
sys.exit(1)
datasetPath = sys.argv[1]
if len(sys.argv) < 3: #No config path sent along
configPath = os.getenv('WMAGENT_CONFIG', None)
else:
configPath = sys.argv[2]
if not os.path.isfile(configPath):
configPath = os.getenv('WMAGENT_CONFIG', None)
if configPath:
config = loadConfigurationFile(configPath)
else:
print "Error! No config could be found"
sys.exit(2)
wmInit = WMInit()
wmInit.setLogging()
wmInit.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
migrateFileBlocks = MigrateFileBlocks(config = config)
migrateFileBlocks.migrateDataset(datasetPath = datasetPath)
class CrabJobCreatorComponent:
"""
_CrabJobCreatorComponent_
"""
################################
# Standard Component Core Methods
################################
def __init__(self, **args):
self.args = {}
self.args.setdefault('Logfile', None)
self.args.setdefault('CacheDir', None)
self.args.setdefault('ProxiesDir', None)
self.args.setdefault('CopyTimeOut', '')
# SE support parameters
# Protocol = local cannot be the default. Any default allowd
# for this parameter... it must be defined from config file.
self.args.setdefault('Protocol', '')
self.args.setdefault('storageName', 'localhost')
self.args.setdefault('storagePort', '')
self.args.setdefault('storagePath', self.args["CacheDir"])
# specific delegation strategy for glExex
self.args.setdefault('glExecDelegation', 'false')
self.args.setdefault('PollInterval',60)
self.args.setdefault("HeartBeatDelay", "00:05:00")
self.args.update(args)
if len(self.args["HeartBeatDelay"]) != 8:
self.HeartBeatDelay="00:05:00"
else:
self.HeartBeatDelay=self.args["HeartBeatDelay"]
# define log file
if self.args['Logfile'] == None:
self.args['Logfile'] = os.path.join(self.args['ComponentDir'],
"ComponentLog")
# create log handler
logHandler = RotatingFileHandler(self.args['Logfile'],
"a", 1000000, 7)
# define log format
logFormatter = logging.Formatter("%(asctime)s:%(message)s")
logHandler.setFormatter(logFormatter)
logging.getLogger().addHandler(logHandler)
logging.getLogger().setLevel(logging.INFO)
## volatile properties
self.wdir = self.args['ComponentDir']
self.maxThreads = int( self.args.get('maxThreads', 5) )
self.timePoolDB = self.args['PollInterval']
# shared sessions
self.blDBsession = BossLiteAPI('MySQL', dbConfig, makePool=True)
self.sessionPool = self.blDBsession.bossLiteDB.getPool()
self.workerPool = self.blDBsession.bossLiteDB.getPool()
# Get configuration
self.init = WMInit()
self.init.setLogging()
self.init.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
self.myThread = threading.currentThread()
self.factory = WMFactory("msgService", "WMCore.MsgService."+ \
self.myThread.dialect)
self.newMsgService = self.myThread.factory['msgService']\
.loadObject("MsgService")
self.ms = MessageService()
self.workerCfg = {}
logging.info(" ")
logging.info("Starting component...")
def startComponent(self):
"""
_startComponent_
Start up the component
"""
# Registration in oldMsgService
self.ms.registerAs("CrabJobCreatorComponent")
self.ms.subscribeTo("JobFailed")
self.ms.subscribeTo("JobSuccess")
self.ms.subscribeTo("CrabJobCreatorComponent:EndDebug")
# Registration in new MsgService
self.myThread.transaction.begin()
self.newMsgService.registerAs("CrabJobCreatorComponent")
self.myThread.transaction.commit()
self.ms.subscribeTo("CrabJobCreatorComponent:HeartBeat")
self.ms.remove("CrabJobCreatorComponent:HeartBeat")
self.ms.publish("CrabJobCreatorComponent:HeartBeat","",self.HeartBeatDelay)
self.ms.commit()
self.workerCfg = self.prepareBaseStatus()
compWMObject = WMObject()
manager = WorkerThreadManager(compWMObject)
manager.addWorker(CrabJobCreatorPoller(self.workerCfg), float(self.timePoolDB))
#######################################
try:
while True:
try:
event, payload = self.ms.get( wait = False )
if event is None:
time.sleep( self.ms.pollTime )
continue
else:
self.__call__(event, payload)
self.ms.commit()
except Exception, exc:
logging.error("ERROR: Problem managing message...")
logging.error(str(exc))
except Exception, e:
logging.error(e)
logging.info(traceback.format_exc())
return
class TestInit:
"""
A set of initialization steps used in many tests.
Test can call the methods from this class to
initialize their default environment so to
minimize code duplication.
"""
def __init__(self, testClassName = "Unknown Class"):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
global hasDatabase
self.hasDatabase = hasDatabase
if self.hasDatabase:
self.init = WMInit()
self.deleteTmp = True
def __del__(self):
if self.deleteTmp:
self.delWorkDir()
self.attemptToCloseDBConnections()
def delWorkDir(self):
if (self.testDir != None):
try:
shutil.rmtree( self.testDir )
except:
# meh, if it fails, I guess something weird happened
pass
def setLogging(self, logLevel = logging.INFO):
"""
Sets logging parameters
"""
# remove old logging instances.
return
logger1 = logging.getLogger()
logger2 = logging.getLogger(self.testClassName)
for logger in [logger1, logger2]:
for handler in logger.handlers:
handler.close()
logger.removeHandler(handler)
self.init.setLogging(self.testClassName, self.testClassName,
logExists = False, logLevel = logLevel)
def generateWorkDir(self, config = None, deleteOnDestruction = True, setTmpDir = False):
self.testDir = tempfile.mkdtemp()
if config:
config.section_("General")
config.General.workDir = self.testDir
os.environ['TESTDIR'] = self.testDir
if os.getenv('WMCORE_KEEP_DIRECTORIES', False):
deleteOnDestruction = True
logging.info("Generated testDir - %s" % self.testDir)
if setTmpDir:
os.environ['TMPDIR'] = self.testDir
self.deleteTmp = deleteOnDestruction
return self.testDir
def getBackendFromDbURL(self, dburl):
dialectPart = dburl.split(":")[0]
if dialectPart == 'mysql':
return 'MySQL'
elif dialectPart == 'sqlite':
return 'SQLite'
elif dialectPart == 'oracle':
return 'Oracle'
elif dialectPart == 'http':
return 'CouchDB'
else:
raise RuntimeError, "Unrecognized dialect %s" % dialectPart
def setDatabaseConnection(self, connectUrl=None, socket=None, destroyAllDatabase = False):
"""
Set up the database connection by retrieving the environment
parameters.
The destroyAllDatabase option is for testing ONLY. Never flip that switch
on in any other instance where you don't know what you're doing.
"""
if not self.hasDatabase:
return
config = self.getConfiguration(connectUrl=connectUrl, socket=socket)
self.coreConfig = config
self.init.setDatabaseConnection(config.CoreDatabase.connectUrl,
config.CoreDatabase.dialect,
config.CoreDatabase.socket)
if trashDatabases or destroyAllDatabase:
self.clearDatabase()
# Have to check whether or not database is empty
# If the database is not empty when we go to set the schema, abort!
result = self.init.checkDatabaseContents()
if len(result) > 0:
msg = "Database not empty, cannot set schema !\n"
msg += str(result)
logging.error(msg)
raise TestInitException(msg)
return
def setSchema(self, customModules = [], useDefault = True, params = None):
"""
Creates the schema in the database for the default
tables/services: trigger, message service, threadpool.
Developers can add their own modules to it using the array
customModules which should follow the proper naming convention.
if useDefault is set to False, it will not instantiate the
schemas in the defaultModules array.
"""
if not self.hasDatabase:
return
defaultModules = ["WMCore.WMBS"]
if not useDefault:
defaultModules = []
# filter out unique modules
modules = {}
for module in (defaultModules + customModules):
modules[module] = 'done'
try:
self.init.setSchema(modules.keys(), params = params)
except Exception, ex:
print traceback.format_exc()
raise ex
# store the list of modules we've added to the DB
modules = {}
for module in (defaultModules + customModules + self.currModules):
modules[module] = 'done'
self.currModules = modules.keys()
return
sender = context.socket(zmq.PUSH)
sender.connect("tcp://localhost:%s" % outPort)
# Build config
if not os.path.exists(configPath):
# We can do nothing -
logging.error("Something in the way of the config path")
sys.exit(1)
f = open(configPath, 'r')
config = cPickle.load(f)
f.close()
# Setup DB
wmInit = WMInit()
setupDB(config, wmInit)
# Create JSON handler
jsonHandler = JSONRequests()
wmFactory = WMFactory(name = "slaveFactory", namespace = namespace)
slaveClass = wmFactory.loadObject(classname = slaveClassName, args = config)
logging.info("Have slave class")
while(True):
encodedInput = receiver.recv()
try:
input = jsonHandler.decode(encodedInput)
def __init__(self, testClassName):
self.testClassName = testClassName
self.testDir = None
self.currModules = []
self.init = WMInit()
self.deleteTmp = True
def testB_Database(self):
"""
_Database_
Testing the database stuff. Only works for MySQL backend
"""
init = WMInit()
url = os.environ.get("DATABASE")
dialect = os.environ.get("DIALECT", "MySQL")
sock = os.environ.get("DBSOCK", None)
init.setDatabaseConnection(url, dialect, sock)
try:
# Initial clear should work
myThread = threading.currentThread()
init.clearDatabase()
# Clear one after another should work
init.setSchema(modules=['WMCore.WMBS'])
init.clearDatabase()
init.setSchema(modules=['WMCore.WMBS'])
init.clearDatabase()
# Clear non-existant DB should work
# Drop the database, and then make sure the database gets recreated
a = myThread.dbi.engine.url.database
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
myThread.dbi.processData("DROP DATABASE %s" % dbName)
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, None)
init.clearDatabase()
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.begin()
myThread.transaction.processData("SELECT * FROM wmbs_job")
init.clearDatabase()
dbName = myThread.dbi.processData(
"SELECT DATABASE() AS dbname")[0].fetchall()[0][0]
self.assertEqual(dbName, a)
myThread.transaction.begin()
init.setSchema(modules=['WMCore.WMBS'])
myThread.transaction.commit()
except:
init.clearDatabase()
raise
init.clearDatabase()
return
def __call__(self, filesetToProcess):
"""
The algorithm itself
"""
# Get configuration
initObj = WMInit()
initObj.setLogging()
initObj.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
myThread = threading.currentThread()
daofactory = DAOFactory(package = "WMCore.WMBS" , \
logger = myThread.logger, \
dbinterface = myThread.dbi)
locationNew = daofactory(classname = "Locations.New")
getFileLoc = daofactory(classname = "Files.GetLocation")
fileInFileset = daofactory(classname = "Files.InFileset")
logging.debug("DBSFeeder is processing %s" % \
filesetToProcess.name)
logging.debug("the filesetBase name %s" \
% (filesetToProcess.name).split(":")[0])
LASTIME = filesetToProcess.lastUpdate
# Get the start Run if asked
startRun = (filesetToProcess.name).split(":")[3]
# get list of files
tries = 1
while True:
try:
blocks = self.dbsReader.getFiles(\
(filesetToProcess.name).split(":")[0])
now = time.time()
logging.debug("DBS queries done ...")
break
except DBSReaderError as ex:
logging.error("DBS error: %s, cannot get files for %s" % \
(str(ex), filesetToProcess.name))
# Close fileset
filesetToProcess.markOpen(False)
return
# connection error, retry
except DbsConnectionError as ex:
logging.error("Unable to connect to DBS, retrying: " + \
str(ex))
if tries > self.connectionAttempts: #too many errors - bail out
return
tries = tries + 1
# check for empty datasets
if blocks == {}:
logging.debug("DBS: Empty blocks - %s" %filesetToProcess.name)
return filesetToProcess
# get all file blocks
blockList = blocks.keys()
# process all file blocks
for fileBlock in blockList:
seList = blocks[fileBlock]['StorageElements']
# add files for non blocked SE
if seList is None or seList == []:
logging.info("fileblock %s - no SE's associated" % \
fileBlock)
continue
else:
for loc in seList:
locationNew.execute(siteName = loc, seName = loc)
for files in blocks[fileBlock]['Files']:
if startRun != 'None':
if len(files['LumiList']):
for lumi in files['LumiList']:
if int(startRun) <= int(lumi['RunNumber' ]):
newfile = File(files['LogicalFileName'], \
size=files['FileSize'], events=files\
['NumberOfEvents'])
LOCK.acquire()
if newfile.exists() == False :
newfile.create()
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate(\
int(time.time()))
filesetToProcess.commit()
runSet = set()
runSet.add(Run( lumi\
['RunNumber' ], *[lumi['LumiSectionNumber']] ))
newfile.addRunSet(runSet)
else:
newfile.loadData()
listFile = fileInFileset.execute\
(filesetToProcess.id)
if {'fileid': newfile[\
'id']} not in listFile:
filesetToProcess.addFile(newfile)
filesetToProcess.setLastUpdate\
(int(time.time()))
filesetToProcess.commit()
val = 0
for run in newfile['runs']:
if lumi['RunNumber' ] == run.run:
val = 1
break
if not val:
runSet = set()
runSet.add(Run(\
lumi['RunNumber' ], *[lumi['LumiSectionNumber']]))
newfile.addRunSet(runSet)
fileLoc = getFileLoc.execute(\
file = files['LogicalFileName'])
if fileLoc:
for loc in seList:
if loc not in fileLoc:
newfile.setLocation(\
loc)
else:
newfile.setLocation(seList)
LOCK.release()
else:
# Assume parents and LumiSection aren't asked
newfile = File(files['LogicalFileName'], \
size=files['FileSize'], events=files['NumberOfEvents'])
LOCK.acquire()
if newfile.exists() == False :
newfile.create()
# Update fileset last update parameter
filesetToProcess.addFile(newfile)
logging.debug("new file created and added by DBS")
filesetToProcess.setLastUpdate(int(time.time()))
filesetToProcess.commit()
else:
newfile.loadData()
listFile = fileInFileset.execute(filesetToProcess.id)
if {'fileid': newfile['id']} not in listFile:
filesetToProcess.addFile(newfile)
logging.debug("new file loaded and added by DBS")
filesetToProcess.setLastUpdate(int(time.time()))
filesetToProcess.commit()
fileLoc = getFileLoc.execute(\
file = files['LogicalFileName'])
if fileLoc:
for loc in seList:
if loc not in fileLoc:
newfile.setLocation(loc)
else:
newfile.setLocation(seList)
LOCK.release()
filesetToProcess.load()
LASTIME = filesetToProcess.lastUpdate
# For re-opned fileset or empty, try until the purge time
if (int(now)/3600 - LASTIME/3600) > self.reopenTime:
filesetToProcess.setLastUpdate(int(time.time()))
filesetToProcess.commit()
if (int(now)/3600 - LASTIME/3600) > self.purgeTime:
filesetToProcess.markOpen(False)
logging.debug("Purge Done...")
filesetToProcess.commit()
logging.debug("DBS feeder work done...")
def __init__(self, **args):
self.args = {}
self.args.setdefault('Logfile', None)
self.args.setdefault('CacheDir', None)
self.args.setdefault('ProxiesDir', None)
self.args.setdefault('CopyTimeOut', '')
# SE support parameters
# Protocol = local cannot be the default. Any default allowd
# for this parameter... it must be defined from config file.
self.args.setdefault('Protocol', '')
self.args.setdefault('storageName', 'localhost')
self.args.setdefault('storagePort', '')
self.args.setdefault('storagePath', self.args["CacheDir"])
# specific delegation strategy for glExex
self.args.setdefault('glExecDelegation', 'false')
self.args.setdefault('PollInterval',60)
self.args.setdefault("HeartBeatDelay", "00:05:00")
self.args.update(args)
if len(self.args["HeartBeatDelay"]) != 8:
self.HeartBeatDelay="00:05:00"
else:
self.HeartBeatDelay=self.args["HeartBeatDelay"]
# define log file
if self.args['Logfile'] == None:
self.args['Logfile'] = os.path.join(self.args['ComponentDir'],
"ComponentLog")
# create log handler
logHandler = RotatingFileHandler(self.args['Logfile'],
"a", 1000000, 7)
# define log format
logFormatter = logging.Formatter("%(asctime)s:%(message)s")
logHandler.setFormatter(logFormatter)
logging.getLogger().addHandler(logHandler)
logging.getLogger().setLevel(logging.INFO)
## volatile properties
self.wdir = self.args['ComponentDir']
self.maxThreads = int( self.args.get('maxThreads', 5) )
self.timePoolDB = self.args['PollInterval']
# shared sessions
self.blDBsession = BossLiteAPI('MySQL', dbConfig, makePool=True)
self.sessionPool = self.blDBsession.bossLiteDB.getPool()
self.workerPool = self.blDBsession.bossLiteDB.getPool()
# Get configuration
self.init = WMInit()
self.init.setLogging()
self.init.setDatabaseConnection(os.getenv("DATABASE"), \
os.getenv('DIALECT'), os.getenv("DBSOCK"))
self.myThread = threading.currentThread()
self.factory = WMFactory("msgService", "WMCore.MsgService."+ \
self.myThread.dialect)
self.newMsgService = self.myThread.factory['msgService']\
.loadObject("MsgService")
self.ms = MessageService()
self.workerCfg = {}
logging.info(" ")
logging.info("Starting component...")
