def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self):
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def __init__(self, config):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
self.dbsUtil = DBSBufferUtil()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.pool = []
self.blocksToCheck = []
self.workInput = None
self.workResult = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup",
"NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType",
"PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload,
"primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url=self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'copyBlock', False)
self.copyPath = getattr(self.config.DBS3Upload, 'copyBlockPath',
'/data/mnorman/block.json')
self.timeoutWaiver = 1
return
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(
self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(
self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName, self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi,
myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(
self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
# use local db for tier0
if hasattr(self.config, "Tier0Feeder"):
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(
centralRequestCouchDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralWMStatsCouchDB = WMStatsWriter(
self.config.AnalyticsDataCollector.centralWMStatsURL)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(
self.config.JobStateMachine.couchurl)
self.dbsBufferUtil = DBSBufferUtil()
if self.pluginName is not None:
pluginFactory = WMFactory(
"plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname=self.pluginName)
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Create some DBSBuffer tables and fake data for testing
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
self.dbsbufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.wmbsFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsUtil = DBSBufferUtil()
# Create two test dbsbuffer workflows
insertWorkflow = self.dbsbufferFactory(classname="InsertWorkflow")
insertWorkflow.execute("Test1", "Task1", 0, 0, 0, 0)
insertWorkflow.execute("Test2", "Task2", 0, 0, 0, 0)
# Update one workflow to "completed" state
updateWorkflow = self.dbsbufferFactory(
classname="UpdateWorkflowsToCompleted")
updateWorkflow.execute(["Test1"])
# Create a test wmbs workflow
testWorkflow = Workflow(spec="somespec.xml",
owner="Erik",
name="Test1",
task="Task1")
testWorkflow.create()
# Create a test dbsbuffer file
self.createTestFiles()
def __init__(self, config):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
self.dbsUtil = DBSBufferUtil()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.pool = []
self.blocksToCheck = []
self.workInput = None
self.workResult = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup", "NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType", "PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload, "primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url=self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'dumpBlock', False)
self.copyPath = os.path.join(getattr(self.config.DBS3Upload, 'componentDir', '/data/srv/'),
'dbsuploader_block.json')
self.timeoutWaiver = 1
return
def __init__(self, config, dbsconfig=None):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
#myThread = threading.currentThread()
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.maxBlockFiles = self.config.DBSUpload.DBSBlockMaxFiles
self.maxBlockTime = self.config.DBSUpload.DBSBlockMaxTime
self.maxBlockSize = self.config.DBSUpload.DBSBlockMaxSize
self.dbsUrl = self.config.DBSUpload.dbsUrl
self.dbsUtil = DBSBufferUtil()
self.pool = []
self.input = None
self.result = None
self.nProc = getattr(self.config.DBSUpload, 'nProcesses', 4)
self.wait = getattr(self.config.DBSUpload, 'dbsWaitTime', 1)
self.nTries = getattr(self.config.DBSUpload, 'dbsNTries', 300)
self.dbs3UploadOnly = getattr(self.config.DBSUpload, 'dbs3UploadOnly',
False)
self.physicsGroup = getattr(self.config.DBSUpload, 'physicsGroup',
'DBS3Test')
self.blockCount = 0
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.dasCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBSUpload, 'copyBlock', False)
self.copyPath = getattr(self.config.DBSUpload, 'copyBlockPath',
'/data/mnorman/block.json')
return
def __init__(self, config, dbsconfig = None):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
self.dbsUtil = DBSBufferUtil()
self.pool = []
self.blocksToCheck = []
self.input = None
self.result = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.dbs3UploadOnly = getattr(self.config.DBS3Upload, "dbs3UploadOnly", False)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup", "NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType", "PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload, "primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url = self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.dasCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'copyBlock', False)
self.copyPath = getattr(self.config.DBS3Upload, 'copyBlockPath',
'/data/mnorman/block.json')
self.timeoutWaiver = 1
return
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
# use local db for tier0
if hasattr(self.config, "Tier0Feeder"):
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.General.centralWMStatsURL)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
self.dbsBufferUtil = DBSBufferUtil()
if self.pluginName is not None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname=self.pluginName)
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Create some DBSBuffer tables and fake data for testing
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
self.dbsbufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.wmbsFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsUtil = DBSBufferUtil()
# Create two test dbsbuffer workflows
insertWorkflow = self.dbsbufferFactory(classname="InsertWorkflow")
insertWorkflow.execute("Test1", "Task1", 0, 0, 0, 0)
insertWorkflow.execute("Test2", "Task2", 0, 0, 0, 0)
# Update one workflow to "completed" state
updateWorkflow = self.dbsbufferFactory(classname="UpdateWorkflowsToCompleted")
updateWorkflow.execute(["Test1"])
# Create a test wmbs workflow
testWorkflow = Workflow(spec="somespec.xml", owner="Erik", name="Test1", task="Task1")
testWorkflow.create()
# Create a test dbsbuffer file
self.createTestFiles()
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
self.condorStates = ("Running", "Idle")
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
results['local_wq_status'] = self.checkLocalWQStatus(
dbname="workqueue")
results['local_wqinbox_status'] = self.checkLocalWQStatus(
dbname="workqueue_inbox")
results['global_wq_status'] = self.checkGlobalWQStatus()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
jobs = self.condorAPI.getCondorJobsSummary()
for state in self.condorStates:
# if there is an error, report it instead of the length of an empty list
if not jobs:
results[state.lower()] = None
else:
results[state.lower()] = int(jobs[0].get(state))
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(
status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def checkLocalWQStatus(self, dbname):
"""
Query local WorkQueue workqueue/workqueue_inbox database to see whether
there are any active elements in this agent.
"""
results = {}
for st in ('Available', 'Negotiating', 'Acquired', 'Running'):
if dbname == "workqueue":
elements = self.localBackend.getElements(status=st,
returnIdOnly=True)
else:
elements = self.localBackend.getInboxElements(
status=st, returnIdOnly=True)
results[st] = len(elements)
return results
def checkGlobalWQStatus(self):
"""
Query Global WorkQueue workqueue database to see whether there are
any active elements set to this agent.
"""
results = {}
for st in ("Acquired", "Running"):
elements = self.globalBackend.getElements(
status=st, returnIdOnly=True, ChildQueueUrl=self.thisAgentUrl)
results[st] = len(elements)
return results
def testBulkLoad(self):
"""
_testBulkLoad_
Can we load in bulk?
"""
addToBuffer = DBSBufferUtil()
bulkLoad = self.daoFactory(classname = "DBSBufferFiles.LoadBulkFilesByID")
testFileChildA = DBSBufferFile(lfn = "/this/is/a/child/lfnA", size = 1024,
events = 20)
testFileChildA.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildB = DBSBufferFile(lfn = "/this/is/a/child/lfnB", size = 1024,
events = 20)
testFileChildB.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildC = DBSBufferFile(lfn = "/this/is/a/child/lfnC", size = 1024,
events = 20)
testFileChildC.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildA.create()
testFileChildB.create()
testFileChildC.create()
testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"])
testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"])
runSet = set()
runSet.add(Run( 1, *[45]))
runSet.add(Run( 2, *[67, 68]))
testFileChildA.addRunSet(runSet)
testFileChildB.addRunSet(runSet)
testFileChildC.addRunSet(runSet)
testFileChildA.save()
testFileChildB.save()
testFileChildC.save()
setCksumAction = self.daoFactory(classname = "DBSBufferFiles.AddChecksumByLFN")
binds = [{'lfn': "/this/is/a/child/lfnA", 'cktype': 'adler32', 'cksum': 201},
{'lfn': "/this/is/a/child/lfnA", 'cktype': 'cksum', 'cksum': 101},
{'lfn': "/this/is/a/child/lfnB", 'cktype': 'adler32', 'cksum': 201},
{'lfn': "/this/is/a/child/lfnB", 'cktype': 'cksum', 'cksum': 101},
{'lfn': "/this/is/a/child/lfnC", 'cktype': 'adler32', 'cksum': 201},
{'lfn': "/this/is/a/child/lfnC", 'cktype': 'cksum', 'cksum': 101}]
setCksumAction.execute(bulkList = binds)
testFile = DBSBufferFile(lfn = "/this/is/a/lfn", size = 1024,
events = 10)
testFile.setAlgorithm(appName = "cmsRun", appVer = "CMSSW_2_1_8",
appFam = "RECO", psetHash = "GIBBERISH",
configContent = "MOREGIBBERISH")
testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFile.create()
testFileChildA.addParents([testFile["lfn"]])
testFileChildB.addParents([testFile["lfn"]])
testFileChildC.addParents([testFile["lfn"]])
binds = [{'id': testFileChildA.exists()},
{'id': testFileChildB.exists()},
{'id': testFileChildC.exists()}]
listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs = binds)
#print listOfFiles
compareList = ['locations', 'psetHash', 'configContent', 'appName',
'appVer', 'appFam', 'events', 'datasetPath', 'runs']
for f in listOfFiles:
self.assertTrue(f['lfn'] in ["/this/is/a/child/lfnA", "/this/is/a/child/lfnB",
"/this/is/a/child/lfnC"],
"Unknown file in loaded results")
self.assertEqual(f['checksums'], {'adler32': '201', 'cksum': '101'})
for parent in f['parents']:
self.assertEqual(parent['lfn'], testFile['lfn'])
for key in compareList:
self.assertEqual(f[key], testFileChildA[key])
class AnalyticsPoller(BaseWorkerThread):
"""
Gether the summary data for request (workflow) from local queue,
local job couchdb, wmbs/boss air and populate summary db for monitoring
"""
def __init__(self, config):
"""
initialize properties specified from config
"""
BaseWorkerThread.__init__(self)
# set the workqueue service for REST call
self.config = config
# need to get campaign, user, owner info
self.agentInfo = initAgentInfo(self.config)
self.summaryLevel = (config.AnalyticsDataCollector.summaryLevel).lower()
self.pluginName = getattr(config.AnalyticsDataCollector, "pluginName", None)
self.plugin = None
def setup(self, parameters):
"""
set db connection(couchdb, wmbs) to prepare to gather information
"""
# set the connection to local queue
if not hasattr(self.config, "Tier0Feeder"):
self.localQueue = WorkQueueService(self.config.AnalyticsDataCollector.localQueueURL)
# set the connection for local couchDB call
self.localCouchDB = LocalCouchDBData(self.config.AnalyticsDataCollector.localCouchURL,
self.config.JobStateMachine.summaryStatsDBName,
self.summaryLevel)
# interface to WMBS/BossAir db
myThread = threading.currentThread()
# set wmagent db data
self.wmagentDB = WMAgentDBData(self.summaryLevel, myThread.dbi, myThread.logger)
# set the connection for local couchDB call
self.localSummaryCouchDB = WMStatsWriter(self.config.AnalyticsDataCollector.localWMStatsURL,
appName="WMStatsAgent")
# use local db for tier0
if hasattr(self.config, "Tier0Feeder"):
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.localT0RequestDBURL
else:
centralRequestCouchDBURL = self.config.AnalyticsDataCollector.centralRequestDBURL
self.centralRequestCouchDB = RequestDBWriter(centralRequestCouchDBURL,
couchapp=self.config.AnalyticsDataCollector.RequestCouchApp)
self.centralWMStatsCouchDB = WMStatsWriter(self.config.General.centralWMStatsURL)
#TODO: change the config to hold couch url
self.localCouchServer = CouchMonitor(self.config.JobStateMachine.couchurl)
self.dbsBufferUtil = DBSBufferUtil()
if self.pluginName is not None:
pluginFactory = WMFactory("plugins", "WMComponent.AnalyticsDataCollector.Plugins")
self.plugin = pluginFactory.loadObject(classname=self.pluginName)
@timeFunction
def algorithm(self, parameters):
"""
get information from wmbs, workqueue and local couch
"""
try:
# jobs per request info
logging.info("Getting Job Couch Data ...")
jobInfoFromCouch = self.localCouchDB.getJobSummaryByWorkflowAndSite()
# fwjr per request info
logging.info("Getting FWJRJob Couch Data ...")
fwjrInfoFromCouch = self.localCouchDB.getJobPerformanceByTaskAndSiteFromSummaryDB()
skippedInfoFromCouch = self.localCouchDB.getSkippedFilesSummaryByWorkflow()
logging.info("Getting Batch Job Data ...")
batchJobInfo = self.wmagentDB.getBatchJobInfo()
logging.info("Getting Finished Task Data ...")
finishedTasks = self.wmagentDB.getFinishedSubscriptionByTask()
logging.info("Getting DBS PhEDEx upload status ...")
completedWfs = self.dbsBufferUtil.getPhEDExDBSStatusForCompletedWorkflows(summary=True)
# get the data from local workqueue:
# request name, input dataset, inWMBS, inQueue
logging.info("Getting Local Queue Data ...")
localQInfo = {}
if not hasattr(self.config, "Tier0Feeder"):
localQInfo = self.localQueue.getAnalyticsData()
else:
logging.debug("Tier-0 instance, not checking WorkQueue")
# combine all the data from 3 sources
logging.info("""Combining data from
Job Couch(%s),
FWJR(%s),
WorkflowsWithSkippedFile(%s),
Batch Job(%s),
Finished Tasks(%s),
Local Queue(%s)
Completed workflows(%s).. ...""",
len(jobInfoFromCouch), len(fwjrInfoFromCouch), len(skippedInfoFromCouch),
len(batchJobInfo), len(finishedTasks), len(localQInfo), len(completedWfs))
tempCombinedData = combineAnalyticsData(jobInfoFromCouch, batchJobInfo)
tempCombinedData2 = combineAnalyticsData(tempCombinedData, localQInfo)
combinedRequests = combineAnalyticsData(tempCombinedData2, completedWfs)
# set the uploadTime - should be the same for all docs
uploadTime = int(time.time())
logging.info("%s requests Data combined,\n uploading request data...", len(combinedRequests))
requestDocs = convertToRequestCouchDoc(combinedRequests, fwjrInfoFromCouch, finishedTasks,
skippedInfoFromCouch, self.agentInfo,
uploadTime, self.summaryLevel)
if self.plugin != None:
self.plugin(requestDocs, self.localSummaryCouchDB, self.centralRequestCouchDB)
existingDocs = self.centralWMStatsCouchDB.getAllAgentRequestRevByID(self.agentInfo["agent_url"])
self.centralWMStatsCouchDB.bulkUpdateData(requestDocs, existingDocs)
logging.info("Request data upload success\n %s request, \nsleep for next cycle", len(requestDocs))
self.centralWMStatsCouchDB.updateAgentInfoInPlace(self.agentInfo["agent_url"],
{"data_last_update": uploadTime, "data_error": "ok"})
except Exception as ex:
msg = str(ex)
logging.exception("Error occurred, will retry later: %s", msg)
try:
self.centralWMStatsCouchDB.updateAgentInfoInPlace(self.agentInfo["agent_url"], {"data_error": msg})
except:
logging.error("upload Agent Info to central couch failed")
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig(dbs3UploadOnly = True)
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBlock(blockName, "malpaquet", 1)
dbsBlock.status = "Open"
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file)
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBlock(blockName, "malpaquet", 1)
dbsBlock.status = "InDBS"
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file)
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def testCloseSettingsPerWorkflow(self):
"""
_testCloseSettingsPerWorkflow_
Test the block closing mechanics in the DBS3 uploader,
this uses a fake dbs api to avoid reliance on external services.
"""
# Signal trapExit that we are a friend
os.environ["DONT_TRAP_EXIT"] = "True"
try:
# Monkey patch the imports of DbsApi
from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
MockDBSUploadPoller.DbsApi = MockDbsApi
# Set the poller and the dbsUtil for verification
myThread = threading.currentThread()
(_, dbsFilePath) = mkstemp(dir = self.testDir)
self.dbsUrl = dbsFilePath
config = self.getConfig()
dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test is event based limits and timeout with no new files.
# Set the files and workflow
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 100,
MaxEvents = 150)
self.createParentFiles(acqEra, nFiles = 20,
workflowName = workflowName,
taskPath = taskPath)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one with less than 150 events.
# On the second iteration the second block is uploaded.
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
self.assertEqual(len(globalFiles), 14)
self.assertEqual(len(notUploadedFiles), 6)
# Check the fake DBS for data
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 2)
for block in fakeDBSInfo:
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['file_count'], 7)
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
time.sleep(3)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 3)
for block in fakeDBSInfo:
if block['block']['file_count'] != 6:
self.assertEqual(block['block']['file_count'], 7)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Now check the limit by size and timeout with new files
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 5,
MaxEvents = 200000000)
self.createParentFiles(acqEra, nFiles = 16,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 6)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertTrue('close_settings' not in block)
self.assertEqual(block['block']['open_for_writing'], 0)
# Put more files, they will go into the same block and then it will be closed
# after timeout
time.sleep(3)
self.createParentFiles(acqEra, nFiles = 3,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 7)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] < 5:
self.assertEqual(block['block']['file_count'], 4)
else:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Finally test size limits
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 1, MaxFiles = 500,
MaxEvents = 200000000, MaxSize = 2048)
self.createParentFiles(acqEra, nFiles = 7,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
time.sleep(2)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 11)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] != 1:
self.assertEqual(block['block']['block_size'], 2048)
self.assertEqual(block['block']['file_count'], 2)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
except:
self.fail("We failed at some point in the test")
finally:
# We don't trust anyone else with _exit
del os.environ["DONT_TRAP_EXIT"]
return
class DBSUploadPoller(BaseWorkerThread):
"""
Handles poll-based DBSUpload
"""
def __init__(self, config):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
# Tier0 Agent don't need this
if hasattr(self.config, "Tier0Feeder"):
self.wmstatsServerSvc = None
else:
wmstatsSvcURL = self.config.General.centralWMStatsURL.replace(
"couchdb/wmstats", "wmstatsserver")
self.wmstatsServerSvc = WMStatsServer(wmstatsSvcURL)
self.dbsUtil = DBSBufferUtil()
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.updateBlocksDAO = daoFactory(classname="UpdateBlocks")
self.updateFilesDAO = daoFactory(classname="UpdateFiles")
self.createBlocksDAO = daoFactory(classname="CreateBlocks")
self.setBlockFilesDAO = daoFactory(classname="SetBlockFiles")
self.pool = []
self.blocksToCheck = []
self.workInput = None
self.workResult = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup",
"NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType",
"PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload,
"primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url=self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'dumpBlock', False)
self.copyPath = os.path.join(
getattr(self.config.DBS3Upload, 'componentDir', '/data/srv/'),
'dbsuploader_block.json')
self.timeoutWaiver = 1
self.datasetParentageCache = {}
return
def setupPool(self):
"""
_setupPool_
Set up the processing pool for work
"""
if self.pool:
# Then something already exists. Continue
return
self.workInput = multiprocessing.Queue()
self.workResult = multiprocessing.Queue()
# Starting up the pool:
for _ in range(self.nProc):
p = multiprocessing.Process(target=uploadWorker,
args=(self.workInput, self.workResult,
self.dbsUrl))
p.start()
self.pool.append(p)
return
def __del__(self):
"""
__del__
Trigger a close of connections if necessary
"""
self.close()
return
def close(self):
"""
_close_
Kill all connections and terminate
"""
terminate = False
for _ in self.pool:
try:
self.workInput.put('STOP')
except Exception as ex:
# Something very strange happens here
# It's like it raises a blank exception
# Upon being told to return
msg = "Hit some exception in deletion\n"
msg += str(ex)
logging.debug(msg)
terminate = True
try:
self.workInput.close()
self.workResult.close()
except Exception:
# What are you going to do?
pass
for proc in self.pool:
if terminate:
proc.terminate()
else:
proc.join()
self.pool = []
self.workInput = None
self.workResult = None
return
def terminate(self, parameters):
"""
Do one more pass, then terminate
"""
logging.debug("terminating. doing one more pass before we die")
self.algorithm(parameters)
@timeFunction
def algorithm(self, parameters=None):
"""
_algorithm_
First, check blocks that may be already uploaded
Then, load blocks
Then, load files
Then, move files into blocks
Then add new blocks in DBSBuffer
Then add blocks to DBS
Then mark blocks as done in DBSBuffer
"""
logging.info("Starting the DBSUpload Polling Cycle")
# refreshing parentageCache every cycle
if self.updateDatasetParentageCache() is False:
return
logging.debug("Dataset parentage map: %s", self.datasetParentageCache)
try:
self.checkBlocks()
self.loadBlocks()
self.loadFiles()
self.checkBlockCompletion()
self.inputBlocks()
self.retrieveBlocks()
except WMException:
raise
except Exception as ex:
msg = "Unhandled Exception in DBSUploadPoller! Error: %s" % str(ex)
logging.exception(msg)
raise DBSUploadException(msg)
def updateDatasetParentageCache(self):
"""
Return True to indicate it successfully fetched the parentage
map. If there was an exception, return False
"""
myThread = threading.currentThread()
success = True
if not self.wmstatsServerSvc:
self.datasetParentageCache = {}
return success
try:
self.datasetParentageCache = self.wmstatsServerSvc.getChildParentDatasetMap(
)
except Exception as ex:
excReason = getattr(ex, 'reason', '')
errorMsg = 'Failed to fetch parentage map from WMStats, skipping this cycle. '
errorMsg += 'Exception: {}. Reason: {}. Error: {}. '.format(
type(ex).__name__, excReason, str(ex))
if isPassiveError(ex):
logging.warning(errorMsg)
else:
errorMsg += 'Hit a terminal exception in DBSUploadPoller.'
raise DBSUploadException(errorMsg)
myThread.logdbClient.post("DBS3Upload_parentMap", errorMsg,
"warning")
success = False
else:
myThread.logdbClient.delete("DBS3Upload_parentMap",
"warning",
this_thread=True)
return success
def loadBlocks(self):
"""
_loadBlocks_
Find all blocks; make sure they're in the cache
"""
openBlocks = self.dbsUtil.findOpenBlocks()
logging.info("Found %d open blocks.", len(openBlocks))
logging.debug("These are the openblocks: %s", openBlocks)
# Load them if we don't have them
blocksToLoad = []
for block in openBlocks:
if block['blockname'] not in self.blockCache:
blocksToLoad.append(block['blockname'])
# Now load the blocks
try:
loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad)
logging.info("Loaded %d blocks.", len(loadedBlocks))
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks to load: %s\n", blocksToLoad)
raise DBSUploadException(msg)
for blockInfo in loadedBlocks:
block = DBSBufferBlock(name=blockInfo['block_name'],
location=blockInfo['origin_site_name'],
datasetpath=blockInfo['datasetpath'])
parent = self.datasetParentageCache.get(blockInfo['datasetpath'])
if parent:
block.addDatasetParent(parent)
logging.debug(
"Load block: Child dataset %s, Parent dataset %s",
blockInfo['datasetpath'], parent)
block.FillFromDBSBuffer(blockInfo)
blockname = block.getName()
# Now we have to load files...
try:
files = self.dbsUtil.loadFilesByBlock(blockname=blockname)
logging.info("Have %i files for block %s", len(files),
blockname)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading files for existing blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks being loaded: %s\n", blockname)
raise DBSUploadException(msg)
# Add the loaded files to the block
for f in files:
block.addFile(f, self.datasetType, self.primaryDatasetType)
# Add to the cache
self.blockCache[blockInfo['block_name']] = block
return
def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded. I will do this by DatasetPath
to break the monstrous calls down into smaller chunks.
"""
dspList = self.dbsUtil.findUploadableDAS()
readyBlocks = []
for dspInfo in dspList:
datasetpath = dspInfo['DatasetPath']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(
datasetpath=datasetpath)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading uploadable files for DatasetPath.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DatasetPath being loaded: %s\n", datasetpath)
raise DBSUploadException(msg)
# Sort the files and blocks by location
fileDict = sortListByKey(loadedFiles, 'locations')
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if files:
currentBlock = self.getBlock(files[0], location, True)
currentBlock.setAcquisitionEra(
era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(
procVer=dspInfo['ProcessingVer'])
for newFile in files:
if not newFile.get('block', 1) is None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile=newFile,
block=currentBlock):
# Then we have to close the block and get a new one
currentBlock.setPendingAndCloseBlock()
readyBlocks.append(currentBlock)
currentBlock = self.getBlock(newFile=newFile,
location=location)
currentBlock.setAcquisitionEra(
era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(
procVer=dspInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(newFile, self.datasetType,
self.primaryDatasetType)
self.filesToUpdate.append({
'filelfn':
newFile['lfn'],
'block':
currentBlock.getName()
})
# Done with the location
readyBlocks.append(currentBlock)
for block in readyBlocks:
self.blockCache[block.getName()] = block
return
def checkBlockCompletion(self):
"""
_checkBlockCompletion_
Mark Open blocks as Pending if they have timed out or their workflows have completed
"""
completedWorkflows = self.dbsUtil.getCompletedWorkflows()
for block in self.blockCache.values():
if block.status == "Open":
if (block.getTime() > block.getMaxBlockTime()) or any(
key in completedWorkflows for key in block.workflows):
block.setPendingAndCloseBlock()
return
def isBlockOpen(self, newFile, block, doTime=False):
"""
_isBlockOpen_
Check and see if a block is full
This will check on time, but that's disabled by default
The plan is to do a time check after we do everything else,
so open blocks about to time out can still get more
files put in them.
"""
if block.getMaxBlockFiles() is None or block.getMaxBlockNumEvents() is None or \
block.getMaxBlockSize() is None or block.getMaxBlockTime() is None:
return True
if block.status != 'Open':
# Then somebody has dumped this already
return False
if block.getSize() + newFile['size'] > block.getMaxBlockSize():
return False
if block.getNumEvents(
) + newFile['events'] > block.getMaxBlockNumEvents():
return False
if block.getNFiles() >= block.getMaxBlockFiles():
# Then we have to dump it because this file
# will put it over the limit.
return False
if block.getTime() > block.getMaxBlockTime() and doTime:
return False
return True
def getBlock(self, newFile, location, skipOpenCheck=False):
"""
_getBlock_
Retrieve a block is one exists with matching datasetpath/location and is open.
If no such block is found create and return a new one.
"""
datasetpath = newFile["datasetPath"]
for block in self.blockCache.values():
if datasetpath == block.getDatasetPath(
) and location == block.getLocation():
if not self.isBlockOpen(newFile=newFile,
block=block) and not skipOpenCheck:
# Block isn't open anymore. Mark it as pending so that it gets uploaded.
block.setPendingAndCloseBlock()
else:
return block
# A suitable open block does not exist. Create a new one.
blockname = "%s#%s" % (datasetpath, makeUUID())
newBlock = DBSBufferBlock(name=blockname,
location=location,
datasetpath=datasetpath)
parent = self.datasetParentageCache.get(datasetpath)
if parent:
newBlock.addDatasetParent(parent)
logging.debug("Get block: Child dataset %s, Parent dataset %s",
datasetpath, parent)
self.blockCache[blockname] = newBlock
return newBlock
def inputBlocks(self):
"""
_inputBlocks_
Loop through all of the "active" blocks and sort them so we can act
appropriately on them. Everything will be sorted based on the
following:
Queued - Block is already being acted on by another process. We just
ignore it.
Pending, not in DBSBuffer - Block that has been closed and needs to
be injected into DBS and also written to DBSBuffer. We'll do both.
Pending, in DBSBuffer - Block has been closed and written to
DBSBuffer. We just need to inject it into DBS.
Open, not in DBSBuffer - Newly created block that needs to be written
not DBSBuffer.
Open, in DBSBuffer - Newly created block that has already been
written to DBSBuffer. We don't have to do anything with it.
"""
if not self.blockCache:
return
myThread = threading.currentThread()
createInDBS = []
createInDBSBuffer = []
updateInDBSBuffer = []
for block in self.blockCache.values():
if block.getName() in self.queuedBlocks:
# Block is already being dealt with by another process. We'll
# ignore it here.
continue
if block.status == 'Pending':
# All pending blocks need to be injected into DBS.
createInDBS.append(block)
# If this is a new block it needs to be added to DBSBuffer
# otherwise it just needs to be updated in DBSBuffer.
if not block.inBuff:
createInDBSBuffer.append(block)
else:
updateInDBSBuffer.append(block)
if block.status == 'Open' and not block.inBuff:
# New block that needs to be added to DBSBuffer.
createInDBSBuffer.append(block)
# First handle new and updated blocks
if createInDBSBuffer or updateInDBSBuffer:
try:
myThread.transaction.begin()
if createInDBSBuffer:
self.createBlocksDAO.execute(
blocks=createInDBSBuffer,
conn=myThread.transaction.conn,
transaction=True)
if updateInDBSBuffer:
self.updateBlocksDAO.execute(
blocks=updateInDBSBuffer,
conn=myThread.transaction.conn,
transaction=True)
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while writing new blocks into DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for DBSBuffer: %s\n", createInDBSBuffer)
logging.debug("Blocks for Update: %s\n", updateInDBSBuffer)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
# Update block status in the block cache. Mark the blocks that we have
# added to DBSBuffer as being in DBSBuffer.
for block in createInDBSBuffer:
self.blockCache.get(block.getName()).inBuff = True
# Record new file/block associations in DBSBuffer.
if self.filesToUpdate:
try:
myThread.transaction.begin()
self.setBlockFilesDAO.execute(binds=self.filesToUpdate,
conn=myThread.transaction.conn,
transaction=True)
self.filesToUpdate = []
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while setting blocks in files.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Files to Update: %s\n", self.filesToUpdate)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
if not createInDBS:
# then there is nothing else to do
return
# Build the pool if it was closed
if not self.pool:
self.setupPool()
# Finally upload blocks to DBS.
for block in createInDBS:
if not block.files:
# What are we doing?
logging.debug("Skipping empty block")
continue
if block.getDataset() is None:
# Then we have to fix the dataset
dbsFile = block.files[0]
block.setDataset(datasetName=dbsFile['datasetPath'],
primaryType=self.primaryDatasetType,
datasetType=self.datasetType,
physicsGroup=dbsFile.get(
'physicsGroup', None),
prep_id=dbsFile.get('prep_id', None))
logging.debug("Found block %s in blocks", block.getName())
block.setPhysicsGroup(group=self.physicsGroup)
encodedBlock = block.convertToDBSBlock()
logging.info("About to insert block %s", block.getName())
self.workInput.put({
'name': block.getName(),
'block': encodedBlock
})
self.blockCount += 1
if self.produceCopy:
with open(self.copyPath, 'w') as jo:
json.dump(encodedBlock, jo, indent=2)
self.queuedBlocks.append(block.getName())
# And all work is in and we're done for now
return
def retrieveBlocks(self):
"""
_retrieveBlocks_
Once blocks are in DBS, we have to retrieve them and see what's
in them. What we do is get everything out of the result queue,
and then update it in DBSBuffer.
To do this, the result queue needs to pass back the blockname
"""
myThread = threading.currentThread()
blocksToClose = []
emptyCount = 0
while self.blockCount > 0:
if emptyCount > self.nTries:
# When timeoutWaiver is 0 raise error.
# It could take long time to get upload data to DBS
# if there are a lot of files are cumulated in the buffer.
# in first try but second try should be faster.
# timeoutWaiver is set as component variable - only resets when component restarted.
# The reason for that is only back log will occur when component is down
# for a long time while other component still running and feeding the data to
# dbsbuffer
if self.timeoutWaiver == 0:
msg = "Exceeded max number of waits while waiting for DBS to finish"
raise DBSUploadException(msg)
else:
self.timeoutWaiver = 0
return
try:
# Get stuff out of the queue with a ridiculously
# short wait time
blockresult = self.workResult.get(timeout=self.wait)
blocksToClose.append(blockresult)
self.blockCount -= 1
logging.debug("Got a block to close")
except queue.Empty:
# This means the queue has no current results
time.sleep(2)
emptyCount += 1
continue
loadedBlocks = []
for result in blocksToClose:
# Remove from list of work being processed
self.queuedBlocks.remove(result.get('name'))
if result["success"] == "uploaded":
block = self.blockCache.get(result.get('name'))
block.status = 'InDBS'
loadedBlocks.append(block)
elif result["success"] == "check":
block = result["name"]
self.blocksToCheck.append(block)
else:
logging.error(
"Error found in multiprocess during process of block %s",
result.get('name'))
logging.error(result['error'])
# Continue to the next block
# Block will remain in pending status until it is transferred
if loadedBlocks:
try:
myThread.transaction.begin()
self.updateFilesDAO.execute(blocks=loadedBlocks,
status="InDBS",
conn=myThread.transaction.conn,
transaction=True)
self.updateBlocksDAO.execute(blocks=loadedBlocks,
conn=myThread.transaction.conn,
transaction=True)
except Exception as ex:
myThread.transaction.rollback()
# possible deadlock with PhEDExInjector, retry once after 10s
logging.warning(
"Oracle exception, possible deadlock due to race condition, retry after 10s sleep"
)
time.sleep(10)
try:
myThread.transaction.begin()
self.updateFilesDAO.execute(blocks=loadedBlocks,
status="InDBS",
conn=myThread.transaction.conn,
transaction=True)
self.updateBlocksDAO.execute(
blocks=loadedBlocks,
conn=myThread.transaction.conn,
transaction=True)
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while finished closed blocks in DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for Update: %s\n", loadedBlocks)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
else:
myThread.transaction.commit()
for block in loadedBlocks:
# Clean things up
name = block.getName()
del self.blockCache[name]
# Clean up the pool so we don't have stuff waiting around
if self.pool:
self.close()
# And we're done
return
def checkBlocks(self):
"""
_checkBlocks_
Check with DBS3 if the blocks marked as check are
uploaded or not.
"""
myThread = threading.currentThread()
blocksUploaded = []
# See if there is anything to check
for block in self.blocksToCheck:
logging.debug("Checking block existence: %s", block)
# Check in DBS if the block was really inserted
try:
result = self.dbsApi.listBlocks(block_name=block)
# it is an empty list if block cannot be found
if result:
loadedBlock = self.blockCache.get(block)
loadedBlock.status = 'InDBS'
blocksUploaded.append(loadedBlock)
except Exception as ex:
msg = "Error trying to check block %s through DBS. Error: %s" % (
block, str(ex))
logging.exception(msg)
# Update the status of those blocks that were truly inserted
if blocksUploaded:
try:
myThread.transaction.begin()
self.updateBlocksDAO.execute(blocks=blocksUploaded,
conn=myThread.transaction.conn,
transaction=True)
self.updateFilesDAO.execute(blocks=blocksUploaded,
status="InDBS",
conn=myThread.transaction.conn,
transaction=True)
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while finished closed blocks in DBSBuffer\n"
msg += str(ex)
logging.exception(msg)
logging.debug("Blocks for Update: %s\n", blocksUploaded)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
for block in blocksUploaded:
# Clean things up
name = block.getName()
del self.blockCache[name]
# Clean the check list
self.blocksToCheck = []
# We're done
return
class DBSBufferUtilTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Create some DBSBuffer tables and fake data for testing
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(
customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
self.dbsbufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.wmbsFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsUtil = DBSBufferUtil()
# Create two test dbsbuffer workflows
insertWorkflow = self.dbsbufferFactory(classname="InsertWorkflow")
insertWorkflow.execute("Test1", "Task1", 0, 0, 0, 0)
insertWorkflow.execute("Test2", "Task2", 0, 0, 0, 0)
# Update one workflow to "completed" state
updateWorkflow = self.dbsbufferFactory(
classname="UpdateWorkflowsToCompleted")
updateWorkflow.execute(["Test1"])
# Create a test wmbs workflow
testWorkflow = Workflow(spec="somespec.xml",
owner="Erik",
name="Test1",
task="Task1")
testWorkflow.create()
# Create a test dbsbuffer file
self.createTestFiles()
def createTestFiles(self):
"""
_createTestFiles_
Create some dbsbuffer test files with different statuses
:return:
"""
phedexStatus = self.dbsbufferFactory(
classname="DBSBufferFiles.SetPhEDExStatus")
for i in range(0, 4):
lfn = "/path/to/some/lfn" + str(i)
# Two files should be InDBS, two files should be NOTUPLOADED
if i in [0, 2]:
status = 'InDBS'
else:
status = 'NOTUPLOADED'
testDBSFile = DBSBufferFile(lfn=lfn,
size=600000,
events=60000,
status=status,
workflowId=1)
testDBSFile.setAlgorithm(appName="cmsRun",
appVer="UNKNOWN",
appFam="RECO",
psetHash="SOMEHASH" + str(i),
configContent="SOMECONTENT")
testDBSFile.setDatasetPath("/path/to/some/dataset")
testDBSFile.create()
# Create all four combinations of status(InDBS,NOTUPLOADED) and in_phedex(0,1)
if i in [0, 1]:
phedexStatus.execute(lfn, 1)
def tearDown(self):
"""
_tearDown_
Drop all the DBSBuffer tables.
"""
self.testInit.clearDatabase()
# List of methods to potentially test
# def loadDBSBufferFilesBulk(self, fileObjs):
# def findUploadableDAS(self):
# def testFindOpenBlocks(self):
# def loadBlocksByDAS(self, das):
#
# def loadBlocks(self, blocknames):
#
# def findUploadableFilesByDAS(self, datasetpath):
#
# def loadFilesByBlock(self, blockname):
def testGetPhEDExDBSStatusForCompletedWorkflows(self):
"""
_testGetPhEDExDBSStatusForCompletedWorkflows_
:return:
"""
results = self.dbsUtil.getPhEDExDBSStatusForCompletedWorkflows()
self.assertEqual(results["Test1"]["InDBS"], 2,
"ERROR: Files with InDBS status is incorrect.")
self.assertEqual(results["Test1"]["InPhEDEx"], 2,
"ERROR: Files with InPhEDEx status is incorrect.")
self.assertEqual(results["Test1"]["NotInDBS"], 2,
"ERROR: Files with NotInDBS status is incorrect.")
self.assertEqual(results["Test1"]["NotInPhEDEx"], 2,
"ERROR: Files with NotInPhEDEx status is incorrect.")
return
def testGetCompletedWorkflows(self):
"""
_testGetCompletedWorkflows_
:return:
"""
results = self.dbsUtil.getCompletedWorkflows()
self.assertEqual(
len(results), 1,
"ERROR: GetCompletedWorkflows returned incorrect number of completed workflows."
)
self.assertIn(
"Test1", results,
"ERROR: GetCompletedWorkflows returned incorrect workflow.")
return
class DBSBufferUtilTest(unittest.TestCase):
def setUp(self):
"""
_setUp_
Setup the database and logging connection. Create some DBSBuffer tables and fake data for testing
"""
self.testInit = TestInit(__file__)
self.testInit.setLogging()
self.testInit.setDatabaseConnection()
self.testInit.setSchema(customModules=["WMComponent.DBS3Buffer", "WMCore.WMBS"],
useDefault=False)
myThread = threading.currentThread()
self.dbsbufferFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.wmbsFactory = DAOFactory(package="WMCore.WMBS",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.dbsUtil = DBSBufferUtil()
# Create two test dbsbuffer workflows
insertWorkflow = self.dbsbufferFactory(classname="InsertWorkflow")
insertWorkflow.execute("Test1", "Task1", 0, 0, 0, 0)
insertWorkflow.execute("Test2", "Task2", 0, 0, 0, 0)
# Update one workflow to "completed" state
updateWorkflow = self.dbsbufferFactory(classname="UpdateWorkflowsToCompleted")
updateWorkflow.execute(["Test1"])
# Create a test wmbs workflow
testWorkflow = Workflow(spec="somespec.xml", owner="Erik", name="Test1", task="Task1")
testWorkflow.create()
# Create a test dbsbuffer file
self.createTestFiles()
def createTestFiles(self):
"""
_createTestFiles_
Create some dbsbuffer test files with different statuses
:return:
"""
phedexStatus = self.dbsbufferFactory(classname="DBSBufferFiles.SetPhEDExStatus")
for i in range(0, 4):
lfn = "/path/to/some/lfn" + str(i)
# Two files should be InDBS, two files should be NOTUPLOADED
if i in [0,2]:
status = 'InDBS'
else:
status = 'NOTUPLOADED'
testDBSFile = DBSBufferFile(lfn=lfn, size=600000, events=60000, status=status, workflowId=1)
testDBSFile.setAlgorithm(appName="cmsRun", appVer="UNKNOWN",
appFam="RECO", psetHash="SOMEHASH" + str(i),
configContent="SOMECONTENT")
testDBSFile.setDatasetPath("/path/to/some/dataset")
testDBSFile.create()
# Create all four combinations of status(InDBS,NOTUPLOADED) and in_phedex(0,1)
if i in [0,1]:
phedexStatus.execute(lfn, 1)
def tearDown(self):
"""
_tearDown_
Drop all the DBSBuffer tables.
"""
self.testInit.clearDatabase()
# List of methods to potentially test
# def loadDBSBufferFilesBulk(self, fileObjs):
# def findUploadableDAS(self):
# def testFindOpenBlocks(self):
# def loadBlocksByDAS(self, das):
#
# def loadBlocks(self, blocknames):
#
# def findUploadableFilesByDAS(self, datasetpath):
#
# def loadFilesByBlock(self, blockname):
def testGetPhEDExDBSStatusForCompletedWorkflows(self):
"""
_testGetPhEDExDBSStatusForCompletedWorkflows_
:return:
"""
results = self.dbsUtil.getPhEDExDBSStatusForCompletedWorkflows()
self.assertEqual(results["Test1"]["InDBS"], 2, "ERROR: Files with InDBS status is incorrect.")
self.assertEqual(results["Test1"]["InPhEDEx"], 2, "ERROR: Files with InPhEDEx status is incorrect.")
self.assertEqual(results["Test1"]["NotInDBS"], 2, "ERROR: Files with NotInDBS status is incorrect.")
self.assertEqual(results["Test1"]["NotInPhEDEx"], 2, "ERROR: Files with NotInPhEDEx status is incorrect.")
return
def testGetCompletedWorkflows(self):
"""
_testGetCompletedWorkflows_
:return:
"""
results = self.dbsUtil.getCompletedWorkflows()
self.assertEqual(len(results), 1, "ERROR: GetCompletedWorkflows returned incorrect number of completed workflows.")
self.assertIn("Test1", results, "ERROR: GetCompletedWorkflows returned incorrect workflow.")
return
def __init__(self, config):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
# Tier0 Agent don't need this
if hasattr(self.config, "Tier0Feeder"):
self.wmstatsServerSvc = None
else:
wmstatsSvcURL = self.config.General.centralWMStatsURL.replace("couchdb/wmstats",
"wmstatsserver")
self.wmstatsServerSvc = WMStatsServer(wmstatsSvcURL)
self.dbsUtil = DBSBufferUtil()
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.updateBlocksDAO = daoFactory(classname="UpdateBlocks")
self.updateFilesDAO = daoFactory(classname="UpdateFiles")
self.createBlocksDAO = daoFactory(classname="CreateBlocks")
self.setBlockFilesDAO = daoFactory(classname="SetBlockFiles")
self.pool = []
self.blocksToCheck = []
self.workInput = None
self.workResult = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup", "NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType", "PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload, "primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url=self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'dumpBlock', False)
self.copyPath = os.path.join(getattr(self.config.DBS3Upload, 'componentDir', '/data/srv/'),
'dbsuploader_block.json')
self.timeoutWaiver = 1
self.datasetParentageCache = {}
return
def testA_basicFunction(self):
"""
_basicFunction_
See if I can make the damn thing work.
"""
myThread = threading.currentThread()
config = self.getConfig()
from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
from dbs.apis.dbsClient import DbsApi
dbsApi = DbsApi(url = config.DBSUpload.dbsUrl)
# This should do nothing
# Just making sure we don't crash
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
name = "ThisIsATest%s" % (int(time.time()))
tier = "RECO"
nFiles = 12
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = "/Cosmics/%s/%s" % (name, tier)
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
time.sleep(5)
# Now look in DBS
try:
result = dbsApi.listDatasets(dataset = datasetPath, detail = True,
dataset_access_type = 'PRODUCTION')
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['data_tier_name'], 'RECO')
self.assertEqual(result[0]['processing_version'], 0)
self.assertEqual(result[0]['acquisition_era_name'], name.split('-')[0])
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 11)
except:
dbsUploader.close()
raise
# All the blocks except for the last one should
# now be there
result = myThread.dbi.processData("SELECT id FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 12)
# The last block should still be open
self.assertEqual(len(dbsUtil.findOpenBlocks()), 1)
try:
dbsUploader.algorithm()
except:
raise
finally:
dbsUploader.close()
# All files should now be available
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 12)
# The last block should now be closed
self.assertEqual(len(dbsUtil.findOpenBlocks()), 0)
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
for res in result:
self.assertEqual(res.values()[0], 'InDBS')
return
def testBulkLoad(self):
"""
_testBulkLoad_
Can we load in bulk?
"""
addToBuffer = DBSBufferUtil()
testFileChildA = DBSBufferFile(lfn="/this/is/a/child/lfnA",
size=1024,
events=20)
testFileChildA.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileChildA.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildB = DBSBufferFile(lfn="/this/is/a/child/lfnB",
size=1024,
events=20)
testFileChildB.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileChildB.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildC = DBSBufferFile(lfn="/this/is/a/child/lfnC",
size=1024,
events=20)
testFileChildC.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFileChildC.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFileChildA.create()
testFileChildB.create()
testFileChildC.create()
testFileChildA.setLocation(["se1.fnal.gov", "se1.cern.ch"])
testFileChildB.setLocation(["se1.fnal.gov", "se1.cern.ch"])
testFileChildC.setLocation(["se1.fnal.gov", "se1.cern.ch"])
runSet = set()
runSet.add(Run(1, *[45]))
runSet.add(Run(2, *[67, 68]))
testFileChildA.addRunSet(runSet)
testFileChildB.addRunSet(runSet)
testFileChildC.addRunSet(runSet)
testFileChildA.save()
testFileChildB.save()
testFileChildC.save()
setCksumAction = self.daoFactory(
classname="DBSBufferFiles.AddChecksumByLFN")
binds = [{
'lfn': "/this/is/a/child/lfnA",
'cktype': 'adler32',
'cksum': 201
}, {
'lfn': "/this/is/a/child/lfnA",
'cktype': 'cksum',
'cksum': 101
}, {
'lfn': "/this/is/a/child/lfnB",
'cktype': 'adler32',
'cksum': 201
}, {
'lfn': "/this/is/a/child/lfnB",
'cktype': 'cksum',
'cksum': 101
}, {
'lfn': "/this/is/a/child/lfnC",
'cktype': 'adler32',
'cksum': 201
}, {
'lfn': "/this/is/a/child/lfnC",
'cktype': 'cksum',
'cksum': 101
}]
setCksumAction.execute(bulkList=binds)
testFile = DBSBufferFile(lfn="/this/is/a/lfn", size=1024, events=10)
testFile.setAlgorithm(appName="cmsRun",
appVer="CMSSW_2_1_8",
appFam="RECO",
psetHash="GIBBERISH",
configContent="MOREGIBBERISH")
testFile.setDatasetPath("/Cosmics/CRUZET09-PromptReco-v1/RECO")
testFile.create()
testFileChildA.addParents([testFile["lfn"]])
testFileChildB.addParents([testFile["lfn"]])
testFileChildC.addParents([testFile["lfn"]])
binds = [{
'id': testFileChildA.exists()
}, {
'id': testFileChildB.exists()
}, {
'id': testFileChildC.exists()
}]
listOfFiles = addToBuffer.loadDBSBufferFilesBulk(fileObjs=binds)
# print listOfFiles
compareList = [
'locations', 'psetHash', 'configContent', 'appName', 'appVer',
'appFam', 'events', 'datasetPath', 'runs'
]
for f in listOfFiles:
self.assertTrue(
f['lfn'] in [
"/this/is/a/child/lfnA", "/this/is/a/child/lfnB",
"/this/is/a/child/lfnC"
], "Unknown file in loaded results")
self.assertEqual(f['checksums'], {
'adler32': '201',
'cksum': '101'
})
for parent in f['parents']:
self.assertEqual(parent['lfn'], testFile['lfn'])
for key in compareList:
self.assertEqual(f[key], testFileChildA[key])
def testA_basicFunction(self):
"""
_basicFunction_
See if I can make the damn thing work.
"""
return
myThread = threading.currentThread()
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
dbsApi = DbsApi(url = config.DBSUpload.dbsUrl)
# This should do nothing
# Just making sure we don't crash
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
name = "ThisIsATest_%s" % (makeUUID())
tier = "RECO"
nFiles = 12
files = self.getFiles(name = name, tier = tier, nFiles = nFiles)
datasetPath = '/%s/%s/%s' % (name, name, tier)
shortPath = '/%s/%s' % (name, name)
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
time.sleep(3)
# Now look in DBS
try:
result = dbsApi.listPrimaryDatasets(dataset = name)
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['primary_ds_name'], name)
result = dbsApi.listDatasets(dataset = datasetPath, detail = True,
dataset_access_type = 'PROCESSING')
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['data_tier_name'], u'RECO')
self.assertEqual(result[0]['processing_version'], u'V0')
self.assertEqual(result[0]['acquisition_era_name'], u"DBS3TEST")
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 11)
except:
dbsUploader.close()
raise
# All the blocks except for the last one should
# now be there
result = myThread.dbi.processData("SELECT id FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 12)
# The last block should still be open
self.assertEqual(len(dbsUtil.findOpenBlocks()), 1)
try:
dbsUploader.algorithm()
except:
raise
finally:
dbsUploader.close()
# All files should now be available
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 12)
# The last block should now be closed
self.assertEqual(len(dbsUtil.findOpenBlocks()), 0)
result = myThread.dbi.processData("SELECT status FROM dbsbuffer_block")[0].fetchall()
for res in result:
self.assertEqual(res.values()[0], 'InDBS')
return
class DBSUploadPoller(BaseWorkerThread):
"""
Handles poll-based DBSUpload
"""
def __init__(self, config):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
self.dbsUtil = DBSBufferUtil()
myThread = threading.currentThread()
self.daoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.pool = []
self.blocksToCheck = []
self.workInput = None
self.workResult = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup", "NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType", "PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload, "primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url=self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'dumpBlock', False)
self.copyPath = os.path.join(getattr(self.config.DBS3Upload, 'componentDir', '/data/srv/'),
'dbsuploader_block.json')
self.timeoutWaiver = 1
return
def setupPool(self):
"""
_setupPool_
Set up the processing pool for work
"""
if len(self.pool) > 0:
# Then something already exists. Continue
return
self.workInput = multiprocessing.Queue()
self.workResult = multiprocessing.Queue()
# Starting up the pool:
for _ in range(self.nProc):
p = multiprocessing.Process(target=uploadWorker,
args=(self.workInput,
self.workResult,
self.dbsUrl))
p.start()
self.pool.append(p)
return
def __del__(self):
"""
__del__
Trigger a close of connections if necessary
"""
self.close()
return
def close(self):
"""
_close_
Kill all connections and terminate
"""
terminate = False
for _ in self.pool:
try:
self.workInput.put('STOP')
except Exception as ex:
# Something very strange happens here
# It's like it raises a blank exception
# Upon being told to return
msg = "Hit some exception in deletion\n"
msg += str(ex)
logging.debug(msg)
terminate = True
try:
self.workInput.close()
self.workResult.close()
except Exception:
# What are you going to do?
pass
for proc in self.pool:
if terminate:
proc.terminate()
else:
proc.join()
self.pool = []
self.workInput = None
self.workResult = None
return
def terminate(self, params):
"""
Do one more pass, then terminate
"""
logging.debug("terminating. doing one more pass before we die")
self.algorithm(params)
@timeFunction
def algorithm(self, parameters=None):
"""
_algorithm_
First, check blocks that may be already uploaded
Then, load blocks
Then, load files
Then, move files into blocks
Then add new blocks in DBSBuffer
Then add blocks to DBS
Then mark blocks as done in DBSBuffer
"""
try:
logging.info("Starting the DBSUpload Polling Cycle")
self.checkBlocks()
self.loadBlocks()
self.loadFiles()
self.checkBlockCompletion()
self.inputBlocks()
self.retrieveBlocks()
except WMException:
raise
except Exception as ex:
msg = "Unhandled Exception in DBSUploadPoller!\n"
msg += str(ex)
msg += str(str(traceback.format_exc()))
logging.error(msg)
raise DBSUploadException(msg)
def loadBlocks(self):
"""
_loadBlocks_
Find all blocks; make sure they're in the cache
"""
openBlocks = self.dbsUtil.findOpenBlocks()
logging.info("Found %d open blocks.", len(openBlocks))
logging.debug("These are the openblocks: %s", openBlocks)
# Load them if we don't have them
blocksToLoad = []
for block in openBlocks:
if not block['blockname'] in self.blockCache.keys():
blocksToLoad.append(block['blockname'])
# Now load the blocks
try:
loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad)
logging.info("Loaded %d blocks.", len(loadedBlocks))
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks to load: %s\n", blocksToLoad)
raise DBSUploadException(msg)
for blockInfo in loadedBlocks:
block = DBSBufferBlock(name=blockInfo['block_name'],
location=blockInfo['origin_site_name'],
datasetpath=blockInfo['datasetpath'])
block.FillFromDBSBuffer(blockInfo)
blockname = block.getName()
# Now we have to load files...
try:
files = self.dbsUtil.loadFilesByBlock(blockname=blockname)
logging.info("Have %i files for block %s", len(files), blockname)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading files for existing blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks being loaded: %s\n", blockname)
raise DBSUploadException(msg)
# Add the loaded files to the block
for f in files:
block.addFile(f, self.datasetType, self.primaryDatasetType)
# Add to the cache
self.blockCache[blockInfo['block_name']] = block
return
def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded. I will do this by DatasetPath
to break the monstrous calls down into smaller chunks.
"""
dspList = self.dbsUtil.findUploadableDAS()
readyBlocks = []
for dspInfo in dspList:
datasetpath = dspInfo['DatasetPath']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(datasetpath=datasetpath)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading uploadable files for DatasetPath.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DatasetPath being loaded: %s\n", datasetpath)
raise DBSUploadException(msg)
# Sort the files and blocks by location
fileDict = sortListByKey(loadedFiles, 'locations')
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if len(files) > 0:
currentBlock = self.getBlock(files[0], location, True)
currentBlock.setAcquisitionEra(era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(procVer=dspInfo['ProcessingVer'])
for newFile in files:
if not newFile.get('block', 1) is None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile=newFile,
block=currentBlock):
# Then we have to close the block and get a new one
currentBlock.setPendingAndCloseBlock()
readyBlocks.append(currentBlock)
currentBlock = self.getBlock(newFile=newFile,
location=location)
currentBlock.setAcquisitionEra(era=dspInfo['AcquisitionEra'])
currentBlock.setProcessingVer(procVer=dspInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(newFile, self.datasetType, self.primaryDatasetType)
self.filesToUpdate.append({'filelfn': newFile['lfn'],
'block': currentBlock.getName()})
# Done with the location
readyBlocks.append(currentBlock)
for block in readyBlocks:
self.blockCache[block.getName()] = block
return
def checkBlockCompletion(self):
"""
_checkBlockCompletion_
Mark Open blocks as Pending if they have timed out or their workflows have completed
"""
completedWorkflows = self.dbsUtil.getCompletedWorkflows()
for block in self.blockCache.values():
if block.status == "Open":
if (block.getTime() > block.getMaxBlockTime()) or any(
key in completedWorkflows for key in block.workflows):
block.setPendingAndCloseBlock()
return
def isBlockOpen(self, newFile, block, doTime=False):
"""
_isBlockOpen_
Check and see if a block is full
This will check on time, but that's disabled by default
The plan is to do a time check after we do everything else,
so open blocks about to time out can still get more
files put in them.
"""
if block.getMaxBlockFiles() is None or block.getMaxBlockNumEvents() is None or \
block.getMaxBlockSize() is None or block.getMaxBlockTime() is None:
return True
if block.status != 'Open':
# Then somebody has dumped this already
return False
if block.getSize() + newFile['size'] > block.getMaxBlockSize():
return False
if block.getNumEvents() + newFile['events'] > block.getMaxBlockNumEvents():
return False
if block.getNFiles() >= block.getMaxBlockFiles():
# Then we have to dump it because this file
# will put it over the limit.
return False
if block.getTime() > block.getMaxBlockTime() and doTime:
return False
return True
def getBlock(self, newFile, location, skipOpenCheck=False):
"""
_getBlock_
Retrieve a block is one exists with matching datasetpath/location and is open.
If no such block is found create and return a new one.
"""
datasetpath = newFile["datasetPath"]
for block in self.blockCache.values():
if datasetpath == block.getDatasetPath() and location == block.getLocation():
if not self.isBlockOpen(newFile=newFile, block=block) and not skipOpenCheck:
# Block isn't open anymore. Mark it as pending so that it gets uploaded.
block.setPendingAndCloseBlock()
else:
return block
# A suitable open block does not exist. Create a new one.
blockname = "%s#%s" % (datasetpath, makeUUID())
newBlock = DBSBufferBlock(name=blockname,
location=location,
datasetpath=datasetpath)
self.blockCache[blockname] = newBlock
return newBlock
def inputBlocks(self):
"""
_inputBlocks_
Loop through all of the "active" blocks and sort them so we can act
appropriately on them. Everything will be sorted based on the
following:
Queued - Block is already being acted on by another process. We just
ignore it.
Pending, not in DBSBuffer - Block that has been closed and needs to
be injected into DBS and also written to DBSBuffer. We'll do both.
Pending, in DBSBuffer - Block has been closed and written to
DBSBuffer. We just need to inject it into DBS.
Open, not in DBSBuffer - Newly created block that needs to be written
not DBSBuffer.
Open, in DBSBuffer - Newly created block that has already been
written to DBSBuffer. We don't have to do anything with it.
"""
myThread = threading.currentThread()
createBlocksDAO = self.daoFactory(classname="CreateBlocks")
updateBlocksDAO = self.daoFactory(classname="UpdateBlocks")
setBlockFilesDAO = self.daoFactory(classname="SetBlockFiles")
createInDBS = []
createInDBSBuffer = []
updateInDBSBuffer = []
for block in self.blockCache.values():
if block.getName() in self.queuedBlocks:
# Block is already being dealt with by another process. We'll
# ignore it here.
continue
if block.status == 'Pending':
# All pending blocks need to be injected into DBS.
createInDBS.append(block)
# If this is a new block it needs to be added to DBSBuffer
# otherwise it just needs to be updated in DBSBuffer.
if not block.inBuff:
createInDBSBuffer.append(block)
else:
updateInDBSBuffer.append(block)
if block.status == 'Open' and not block.inBuff:
# New block that needs to be added to DBSBuffer.
createInDBSBuffer.append(block)
# Build the pool if it was closed
if len(self.pool) == 0:
self.setupPool()
# First handle new and updated blocks
if len(createInDBSBuffer) > 0 or len(updateInDBSBuffer) > 0:
try:
myThread.transaction.begin()
if len(createInDBSBuffer) > 0:
createBlocksDAO.execute(blocks=createInDBSBuffer,
conn=myThread.transaction.conn,
transaction=True)
if len(updateInDBSBuffer) > 0:
updateBlocksDAO.execute(blocks=updateInDBSBuffer,
conn=myThread.transaction.conn,
transaction=True)
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while writing new blocks into DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for DBSBuffer: %s\n", createInDBSBuffer)
logging.debug("Blocks for Update: %s\n", updateInDBSBuffer)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
# Update block status in the block cache. Mark the blocks that we have
# added to DBSBuffer as being in DBSBuffer.
for block in createInDBSBuffer:
self.blockCache.get(block.getName()).inBuff = True
# Record new file/block associations in DBSBuffer.
if len(self.filesToUpdate) > 0:
try:
myThread.transaction.begin()
setBlockFilesDAO.execute(binds=self.filesToUpdate,
conn=myThread.transaction.conn,
transaction=True)
self.filesToUpdate = []
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while setting blocks in files.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Files to Update: %s\n", self.filesToUpdate)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
# Finally upload blocks to DBS.
for block in createInDBS:
if len(block.files) < 1:
# What are we doing?
logging.debug("Skipping empty block")
continue
if block.getDataset() == None:
# Then we have to fix the dataset
dbsFile = block.files[0]
block.setDataset(datasetName=dbsFile['datasetPath'],
primaryType=self.primaryDatasetType,
datasetType=self.datasetType,
physicsGroup=dbsFile.get('physicsGroup', None),
prep_id=dbsFile.get('prep_id', None))
logging.debug("Found block %s in blocks", block.getName())
block.setPhysicsGroup(group=self.physicsGroup)
encodedBlock = block.convertToDBSBlock()
logging.info("About to insert block %s", block.getName())
self.workInput.put({'name': block.getName(), 'block': encodedBlock})
self.blockCount += 1
if self.produceCopy:
with open(self.copyPath, 'w') as jo:
json.dump(encodedBlock, jo, indent=2)
self.queuedBlocks.append(block.getName())
# And all work is in and we're done for now
return
def retrieveBlocks(self):
"""
_retrieveBlocks_
Once blocks are in DBS, we have to retrieve them and see what's
in them. What we do is get everything out of the result queue,
and then update it in DBSBuffer.
To do this, the result queue needs to pass back the blockname
"""
myThread = threading.currentThread()
updateBlocksDAO = self.daoFactory(classname="UpdateBlocks")
updateFilesDAO = self.daoFactory(classname="UpdateFiles")
blocksToClose = []
emptyCount = 0
while self.blockCount > 0:
if emptyCount > self.nTries:
# When timeoutWaiver is 0 raise error.
# It could take long time to get upload data to DBS
# if there are a lot of files are cumulated in the buffer.
# in first try but second try should be faster.
# timeoutWaiver is set as component variable - only resets when component restarted.
# The reason for that is only back log will occur when component is down
# for a long time while other component still running and feeding the data to
# dbsbuffer
if self.timeoutWaiver == 0:
msg = "Exceeded max number of waits while waiting for DBS to finish"
raise DBSUploadException(msg)
else:
self.timeoutWaiver = 0
return
try:
# Get stuff out of the queue with a ridiculously
# short wait time
blockresult = self.workResult.get(timeout=self.wait)
blocksToClose.append(blockresult)
self.blockCount -= 1
logging.debug("Got a block to close")
except Queue.Empty:
# This means the queue has no current results
time.sleep(2)
emptyCount += 1
continue
loadedBlocks = []
for result in blocksToClose:
# Remove from list of work being processed
self.queuedBlocks.remove(result.get('name'))
if result["success"] == "uploaded":
block = self.blockCache.get(result.get('name'))
block.status = 'InDBS'
loadedBlocks.append(block)
elif result["success"] == "check":
block = result["name"]
self.blocksToCheck.append(block)
else:
logging.error("Error found in multiprocess during process of block %s", result.get('name'))
logging.error(result['error'])
# Continue to the next block
# Block will remain in pending status until it is transferred
if len(loadedBlocks) > 0:
try:
myThread.transaction.begin()
updateFilesDAO.execute(blocks=loadedBlocks, status="InDBS",
conn=myThread.transaction.conn,
transaction=True)
updateBlocksDAO.execute(blocks=loadedBlocks,
conn=myThread.transaction.conn,
transaction=True)
except Exception as ex:
myThread.transaction.rollback()
# possible deadlock with PhEDExInjector, retry once after 10s
logging.warning("Oracle exception, possible deadlock due to race condition, retry after 10s sleep")
time.sleep(10)
try:
myThread.transaction.begin()
updateFilesDAO.execute(blocks=loadedBlocks, status="InDBS",
conn=myThread.transaction.conn,
transaction=True)
updateBlocksDAO.execute(blocks=loadedBlocks,
conn=myThread.transaction.conn,
transaction=True)
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while finished closed blocks in DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for Update: %s\n", loadedBlocks)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
else:
myThread.transaction.commit()
for block in loadedBlocks:
# Clean things up
name = block.getName()
del self.blockCache[name]
# Clean up the pool so we don't have stuff waiting around
if len(self.pool) > 0:
self.close()
# And we're done
return
def checkBlocks(self):
"""
_checkBlocks_
Check with DBS3 if the blocks marked as check are
uploaded or not.
"""
myThread = threading.currentThread()
updateBlocksDAO = self.daoFactory(classname="UpdateBlocks")
updateFilesDAO = self.daoFactory(classname="UpdateFiles")
blocksUploaded = []
# See if there is anything to check
for block in self.blocksToCheck:
logging.debug("Checking block existence: %s", block)
# Check in DBS if the block was really inserted
try:
result = self.dbsApi.listBlocks(block_name=block)
for blockResult in result:
if blockResult['block_name'] == block:
loadedBlock = self.blockCache.get(block)
loadedBlock.status = 'InDBS'
blocksUploaded.append(loadedBlock)
break
except Exception as ex:
exString = str(ex)
msg = "Error trying to check block %s through DBS.\n" % block
msg += exString
logging.error(msg)
logging.error(str(traceback.format_exc()))
# Update the status of those blocks that were truly inserted
if len(blocksUploaded) > 0:
try:
myThread.transaction.begin()
updateBlocksDAO.execute(blocks=blocksUploaded,
conn=myThread.transaction.conn,
transaction=True)
updateFilesDAO.execute(blocks=blocksUploaded, status="InDBS",
conn=myThread.transaction.conn,
transaction=True)
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
myThread.transaction.rollback()
msg = "Unhandled exception while finished closed blocks in DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for Update: %s\n", blocksUploaded)
raise DBSUploadException(msg)
else:
myThread.transaction.commit()
for block in blocksUploaded:
# Clean things up
name = block.getName()
del self.blockCache[name]
# Clean the check list
self.blocksToCheck = []
# We're done
return
def testA_basicFunction(self):
"""
_basicFunction_
See if I can make the damn thing work.
"""
myThread = threading.currentThread()
config = self.getConfig()
from WMComponent.DBS3Buffer.DBSUploadPoller import DBSUploadPoller
dbsUploader = DBSUploadPoller(config=config)
dbsUtil = DBSBufferUtil()
from dbs.apis.dbsClient import DbsApi
dbsApi = DbsApi(url=config.DBSUpload.dbsUrl)
# This should do nothing
# Just making sure we don't crash
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
name = "ThisIsATest%s" % (int(time.time()))
tier = "RECO"
nFiles = 12
name = name.replace('-', '_')
name = '%s-v0' % name
files = self.getFiles(name=name, tier=tier, nFiles=nFiles)
datasetPath = "/Cosmics/%s/%s" % (name, tier)
try:
dbsUploader.algorithm()
except:
dbsUploader.close()
raise
time.sleep(5)
# Now look in DBS
try:
result = dbsApi.listDatasets(dataset=datasetPath,
detail=True,
dataset_access_type='PRODUCTION')
self.assertEqual(len(result), 1)
self.assertEqual(result[0]['data_tier_name'], 'RECO')
self.assertEqual(result[0]['processing_version'], 0)
self.assertEqual(result[0]['acquisition_era_name'],
name.split('-')[0])
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 11)
except:
dbsUploader.close()
raise
# All the blocks except for the last one should
# now be there
result = myThread.dbi.processData(
"SELECT id FROM dbsbuffer_block")[0].fetchall()
self.assertEqual(len(result), 12)
# The last block should still be open
self.assertEqual(len(dbsUtil.findOpenBlocks()), 1)
try:
dbsUploader.algorithm()
except:
raise
finally:
dbsUploader.close()
# All files should now be available
result = dbsApi.listFiles(dataset=datasetPath)
self.assertEqual(len(result), 12)
# The last block should now be closed
self.assertEqual(len(dbsUtil.findOpenBlocks()), 0)
result = myThread.dbi.processData(
"SELECT status FROM dbsbuffer_block")[0].fetchall()
for res in result:
self.assertEqual(res.values()[0], 'InDBS')
return
def testDualUpload(self):
"""
_testDualUpload_
Verify that the dual upload mode works correctly.
"""
self.dbsApi = DbsApi(url = self.dbsUrl)
config = self.getConfig()
dbsUploader = DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test verifies that uploader will poll and then not do anything
# as the database is empty.
dbsUploader.algorithm()
acqEra = "Summer%s" % (int(time.time()))
parentFiles = self.createParentFiles(acqEra)
(moreParentFiles, childFiles) = \
self.createFilesWithChildren(parentFiles, acqEra)
allFiles = parentFiles + moreParentFiles
allBlocks = []
for i in range(4):
DBSBufferDataset(parentFiles[0]["datasetPath"]).create()
blockName = parentFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "Open"
dbsBlock.setDataset(parentFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in allFiles[i * 5 : (i * 5) + 5]:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
if i < 2:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
allBlocks.append(dbsBlock)
DBSBufferDataset(childFiles[0]["datasetPath"]).create()
blockName = childFiles[0]["datasetPath"] + "#" + makeUUID()
dbsBlock = DBSBufferBlock(blockName,
location = "malpaquet",
datasetpath = None)
dbsBlock.status = "InDBS"
dbsBlock.setDataset(childFiles[0]["datasetPath"], 'data', 'VALID')
dbsUtil.createBlocks([dbsBlock])
for file in childFiles:
dbsBlock.addFile(file, 'data', 'VALID')
dbsUtil.setBlockFiles({"block": blockName, "filelfn": file["lfn"]})
dbsUtil.updateFileStatus([dbsBlock], "InDBS")
dbsUploader.algorithm()
time.sleep(5)
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles)
# Change the status of the rest of the parent blocks so we can upload
# them and the children.
for dbsBlock in allBlocks:
dbsBlock.status = "InDBS"
dbsUtil.updateBlocks([dbsBlock])
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(parentFiles[0]["datasetPath"], parentFiles + moreParentFiles)
# Run the uploader one more time to upload the children.
dbsUploader.algorithm()
time.sleep(5)
self.verifyData(childFiles[0]["datasetPath"], childFiles)
return
def __init__(self):
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def testCloseSettingsPerWorkflow(self):
"""
_testCloseSettingsPerWorkflow_
Test the block closing mechanics in the DBS3 uploader,
this uses a fake dbs api to avoid reliance on external services.
"""
# Signal trapExit that we are a friend
os.environ["DONT_TRAP_EXIT"] = "True"
try:
# Monkey patch the imports of DbsApi
from WMComponent.DBS3Buffer import DBSUploadPoller as MockDBSUploadPoller
MockDBSUploadPoller.DbsApi = MockDbsApi
# Set the poller and the dbsUtil for verification
myThread = threading.currentThread()
(_, dbsFilePath) = mkstemp(dir = self.testDir)
self.dbsUrl = dbsFilePath
config = self.getConfig()
dbsUploader = MockDBSUploadPoller.DBSUploadPoller(config = config)
dbsUtil = DBSBufferUtil()
# First test is event based limits and timeout with no new files.
# Set the files and workflow
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 100,
MaxEvents = 150)
self.createParentFiles(acqEra, nFiles = 20,
workflowName = workflowName,
taskPath = taskPath)
# The algorithm needs to be run twice. On the first iteration it will
# create all the blocks and upload one with less than 150 events.
# On the second iteration the second block is uploaded.
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
globalFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'InDBS'")[0].fetchall()
notUploadedFiles = myThread.dbi.processData("SELECT id FROM dbsbuffer_file WHERE status = 'NOTUPLOADED'")[0].fetchall()
self.assertEqual(len(globalFiles), 14)
self.assertEqual(len(notUploadedFiles), 6)
# Check the fake DBS for data
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 2)
for block in fakeDBSInfo:
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['file_count'], 7)
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
time.sleep(3)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 3)
for block in fakeDBSInfo:
if block['block']['file_count'] != 6:
self.assertEqual(block['block']['file_count'], 7)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Now check the limit by size and timeout with new files
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 2, MaxFiles = 5,
MaxEvents = 200000000)
self.createParentFiles(acqEra, nFiles = 16,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 1)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 6)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertTrue('close_settings' not in block)
self.assertEqual(block['block']['open_for_writing'], 0)
# Put more files, they will go into the same block and then it will be closed
# after timeout
time.sleep(3)
self.createParentFiles(acqEra, nFiles = 3,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
openBlocks = dbsUtil.findOpenBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 7)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] < 5:
self.assertEqual(block['block']['file_count'], 4)
else:
self.assertEqual(block['block']['file_count'], 5)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
# Finally test size limits
acqEra = "TropicalSeason%s" % (int(time.time()))
workflowName = 'TestWorkload%s' % (int(time.time()))
taskPath = '/%s/TestProcessing' % workflowName
self.injectWorkflow(workflowName, taskPath,
MaxWaitTime = 1, MaxFiles = 500,
MaxEvents = 200000000, MaxSize = 2048)
self.createParentFiles(acqEra, nFiles = 7,
workflowName = workflowName,
taskPath = taskPath)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
time.sleep(2)
dbsUploader.algorithm()
dbsUploader.checkBlocks()
self.assertEqual(len(openBlocks), 0)
fakeDBS = open(self.dbsUrl, 'r')
fakeDBSInfo = json.load(fakeDBS)
fakeDBS.close()
self.assertEqual(len(fakeDBSInfo), 11)
for block in fakeDBSInfo:
if acqEra in block['block']['block_name']:
if block['block']['file_count'] != 1:
self.assertEqual(block['block']['block_size'], 2048)
self.assertEqual(block['block']['file_count'], 2)
self.assertTrue('block_events' not in block['block'])
self.assertEqual(block['block']['open_for_writing'], 0)
self.assertTrue('close_settings' not in block)
except:
self.fail("We failed at some point in the test")
finally:
# We don't trust anyone else with _exit
del os.environ["DONT_TRAP_EXIT"]
return
class DrainStatusAPI(object):
"""
Provides methods for querying dbs and condor for drain statistics
"""
def __init__(self, config):
# queue url used in WorkQueueManager
self.thisAgentUrl = "http://" + config.Agent.hostName + ":5984"
self.globalBackend = WorkQueueBackend(config.WorkloadSummary.couchurl)
self.localBackend = WorkQueueBackend(config.WorkQueueManager.couchurl)
self.dbsUtil = DBSBufferUtil()
self.condorAPI = PyCondorAPI()
def collectDrainInfo(self):
"""
Call methods to check the drain status
"""
results = {}
results['workflows_completed'] = self.checkWorkflows()
# if workflows are completed, collect additional drain statistics
if results['workflows_completed']:
results['upload_status'] = self.checkFileUploadStatus()
results['condor_status'] = self.checkCondorStates()
results['local_wq_status'] = self.checkLocalWQStatus(dbname="workqueue")
results['local_wqinbox_status'] = self.checkLocalWQStatus(dbname="workqueue_inbox")
results['global_wq_status'] = self.checkGlobalWQStatus()
return results
def checkWorkflows(self):
"""
Check to see if all workflows have a 'completed' status
"""
results = self.dbsUtil.isAllWorkflowCompleted()
return results
def checkCondorStates(self):
"""
Check idle and running jobs in Condor
"""
results = {}
queries = [["1", "idle"], ["2", "running"]]
for query in queries:
jobs = self.condorAPI.getCondorJobs("JobStatus=="+query[0], [])
# if there is an error, report it instead of the length of an empty list
if jobs is None:
results[query[1]] = "unknown (schedd query error)"
else:
results[query[1]] = len(jobs)
return results
def checkFileUploadStatus(self):
"""
Check file upload status:
Blocks open in DBS
Files not uploaded in DBS
Files not uploaded to Phedex
"""
results = {}
results['dbs_open_blocks'] = self.dbsUtil.countOpenBlocks()
results['dbs_notuploaded'] = self.dbsUtil.countFilesByStatus(status="NOTUPLOADED")
results['phedex_notuploaded'] = self.dbsUtil.countPhedexNotUploaded()
return results
def checkLocalWQStatus(self, dbname):
"""
Query local WorkQueue workqueue/workqueue_inbox database to see whether
there are any active elements in this agent.
"""
results = {}
for st in ('Available', 'Negotiating', 'Acquired', 'Running'):
if dbname == "workqueue":
elements = self.localBackend.getElements(status=st, returnIdOnly=True)
else:
elements = self.localBackend.getInboxElements(status=st, returnIdOnly=True)
results[st] = len(elements)
return results
def checkGlobalWQStatus(self):
"""
Query Global WorkQueue workqueue database to see whether there are
any active elements set to this agent.
"""
results = {}
for st in ("Acquired", "Running"):
elements = self.globalBackend.getElements(status=st, returnIdOnly=True,
ChildQueueUrl=self.thisAgentUrl)
results[st] = len(elements)
return results
def __init__(self, config):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
# Tier0 Agent don't need this
if hasattr(self.config, "Tier0Feeder"):
self.wmstatsServerSvc = None
else:
wmstatsSvcURL = self.config.General.centralWMStatsURL.replace(
"couchdb/wmstats", "wmstatsserver")
self.wmstatsServerSvc = WMStatsServer(wmstatsSvcURL)
self.dbsUtil = DBSBufferUtil()
myThread = threading.currentThread()
daoFactory = DAOFactory(package="WMComponent.DBS3Buffer",
logger=myThread.logger,
dbinterface=myThread.dbi)
self.updateBlocksDAO = daoFactory(classname="UpdateBlocks")
self.updateFilesDAO = daoFactory(classname="UpdateFiles")
self.createBlocksDAO = daoFactory(classname="CreateBlocks")
self.setBlockFilesDAO = daoFactory(classname="SetBlockFiles")
self.pool = []
self.blocksToCheck = []
self.workInput = None
self.workResult = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup",
"NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType",
"PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload,
"primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url=self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'dumpBlock', False)
self.copyPath = os.path.join(
getattr(self.config.DBS3Upload, 'componentDir', '/data/srv/'),
'dbsuploader_block.json')
self.timeoutWaiver = 1
self.datasetParentageCache = {}
return
class DBSUploadPoller(BaseWorkerThread):
"""
Handles poll-based DBSUpload
"""
def __init__(self, config, dbsconfig = None):
"""
Initialise class members
"""
logging.info("Running __init__ for DBS3 Uploader")
BaseWorkerThread.__init__(self)
self.config = config
# This is slightly dangerous, but DBSUpload depends
# on DBSInterface anyway
self.dbsUrl = self.config.DBS3Upload.dbsUrl
self.dbsUtil = DBSBufferUtil()
self.pool = []
self.blocksToCheck = []
self.input = None
self.result = None
self.nProc = getattr(self.config.DBS3Upload, 'nProcesses', 4)
self.wait = getattr(self.config.DBS3Upload, 'dbsWaitTime', 2)
self.nTries = getattr(self.config.DBS3Upload, 'dbsNTries', 300)
self.dbs3UploadOnly = getattr(self.config.DBS3Upload, "dbs3UploadOnly", False)
self.physicsGroup = getattr(self.config.DBS3Upload, "physicsGroup", "NoGroup")
self.datasetType = getattr(self.config.DBS3Upload, "datasetType", "PRODUCTION")
self.primaryDatasetType = getattr(self.config.DBS3Upload, "primaryDatasetType", "mc")
self.blockCount = 0
self.dbsApi = DbsApi(url = self.dbsUrl)
# List of blocks currently in processing
self.queuedBlocks = []
# Set up the pool of worker processes
self.setupPool()
# Setting up any cache objects
self.blockCache = {}
self.dasCache = {}
self.filesToUpdate = []
self.produceCopy = getattr(self.config.DBS3Upload, 'copyBlock', False)
self.copyPath = getattr(self.config.DBS3Upload, 'copyBlockPath',
'/data/mnorman/block.json')
self.timeoutWaiver = 1
return
def setupPool(self):
"""
_setupPool_
Set up the processing pool for work
"""
if len(self.pool) > 0:
# Then something already exists. Continue
return
self.input = multiprocessing.Queue()
self.result = multiprocessing.Queue()
# Starting up the pool:
for _ in range(self.nProc):
p = multiprocessing.Process(target = uploadWorker,
args = (self.input,
self.result, self.dbsUrl))
p.start()
self.pool.append(p)
return
def __del__(self):
"""
__del__
Trigger a close of connections if necessary
"""
self.close()
return
def close(self):
"""
_close_
Kill all connections and terminate
"""
terminate = False
for _ in self.pool:
try:
self.input.put('STOP')
except Exception as ex:
# Something very strange happens here
# It's like it raises a blank exception
# Upon being told to return
msg = "Hit some exception in deletion\n"
msg += str(ex)
logging.debug(msg)
terminate = True
try:
self.input.close()
self.result.close()
except:
# What are you going to do?
pass
for proc in self.pool:
if terminate:
proc.terminate()
else:
proc.join()
self.pool = []
self.input = None
self.result = None
return
def terminate(self, params):
"""
Do one more pass, then terminate
"""
logging.debug("terminating. doing one more pass before we die")
self.algorithm(params)
def algorithm(self, parameters = None):
"""
_algorithm_
First, check blocks that may be already uploaded
Then, load blocks
Then, load files
Then, move files into blocks
Then add new blocks in DBSBuffer
Then add blocks to DBS
Then mark blocks as done in DBSBuffer
"""
try:
logging.info("Starting the DBSUpload Polling Cycle")
self.checkBlocks()
self.loadBlocks()
# The following two functions will actually place new files into
# blocks. In DBS3 upload mode we rely on something else to do that
# for us and will skip this step.
if not self.dbs3UploadOnly:
self.loadFiles()
self.checkTimeout()
self.checkCompleted()
self.inputBlocks()
self.retrieveBlocks()
except WMException:
raise
except Exception as ex:
msg = "Unhandled Exception in DBSUploadPoller!\n"
msg += str(ex)
msg += str(str(traceback.format_exc()))
logging.error(msg)
raise DBSUploadException(msg)
def loadBlocks(self):
"""
_loadBlocks_
Find all blocks; make sure they're in the cache
"""
openBlocks = self.dbsUtil.findOpenBlocks(self.dbs3UploadOnly)
logging.info("These are the openblocks: %s" % openBlocks)
# Load them if we don't have them
blocksToLoad = []
for block in openBlocks:
if not block['blockname'] in self.blockCache.keys():
blocksToLoad.append(block['blockname'])
# Now load the blocks
try:
loadedBlocks = self.dbsUtil.loadBlocks(blocksToLoad, self.dbs3UploadOnly)
logging.info("Loaded blocks: %s" % loadedBlocks)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks to load: %s\n" % blocksToLoad)
raise DBSUploadException(msg)
for blockInfo in loadedBlocks:
das = blockInfo['DatasetAlgo']
loc = blockInfo['origin_site_name']
workflow = blockInfo['workflow']
block = DBSBlock(name = blockInfo['block_name'],
location = loc, das = das, workflow = workflow)
block.FillFromDBSBuffer(blockInfo)
blockname = block.getName()
# Now we have to load files...
try:
files = self.dbsUtil.loadFilesByBlock(blockname = blockname)
logging.info("Have %i files for block %s" % (len(files), blockname))
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading files for existing blocks.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks being loaded: %s\n" % blockname)
raise DBSUploadException(msg)
# Add the loaded files to the block
for file in files:
block.addFile(file, self.datasetType, self.primaryDatasetType)
# Add to the cache
self.addNewBlock(block = block)
# All blocks should now be loaded and present
# in both the block cache (which has all the info)
# and the dasCache (which is a list of name pointers
# to the keys in the block cache).
return
def loadFiles(self):
"""
_loadFiles_
Load all files that need to be loaded. I will do this by DAS for now to
break the monstrous calls down into smaller chunks.
"""
# Grab all the Dataset-Algo combindations
dasList = self.dbsUtil.findUploadableDAS()
if len(dasList) < 1:
# Then there's nothing to do
return []
readyBlocks = []
for dasInfo in dasList:
dasID = dasInfo['DAS_ID']
# Get the files
try:
loadedFiles = self.dbsUtil.findUploadableFilesByDAS(das = dasID)
except WMException:
raise
except Exception as ex:
msg = "Unhandled exception while loading uploadable files for DAS.\n"
msg += str(ex)
logging.error(msg)
logging.debug("DAS being loaded: %s\n" % dasID)
raise DBSUploadException(msg)
# Sort the files and blocks by location
fileDict = sortListByKey(input = loadedFiles, key = 'locations')
# Now add each file
for location in fileDict.keys():
files = fileDict.get(location)
if len(files) < 1:
# Nothing to do here
continue
currentBlock = self.getBlock(files[0], location, dasID, True)
currentBlock.setAcquisitionEra(era = dasInfo['AcquisitionEra'])
currentBlock.setProcessingVer(procVer = dasInfo['ProcessingVer'])
for newFile in files:
if not newFile.get('block', 1) == None:
# Then this file already has a block
# It should be accounted for somewhere
# Or loaded with the block
continue
# Check if we can put files in this block
if not self.isBlockOpen(newFile = newFile,
block = currentBlock):
# Then we have to close the block and get a new one
currentBlock.setPendingAndCloseBlock()
readyBlocks.append(currentBlock)
currentBlock = self.getBlock(newFile = newFile,
location = location,
das = dasID)
currentBlock.setAcquisitionEra(era = dasInfo['AcquisitionEra'])
currentBlock.setProcessingVer(procVer = dasInfo['ProcessingVer'])
# Now deal with the file
currentBlock.addFile(newFile, self.datasetType, self.primaryDatasetType)
self.filesToUpdate.append({'filelfn': newFile['lfn'],
'block': currentBlock.getName()})
# Done with the location
readyBlocks.append(currentBlock)
# Should be done with the DAS once we've added all files
# Update the blockCache with what is now ready.
for block in readyBlocks:
self.blockCache[block.getName()] = block
return
def checkTimeout(self):
"""
_checkTimeout_
Loop all Open blocks and mark them as Pending if they have timed out.
"""
for block in self.blockCache.values():
if block.status == "Open" and block.getTime() > block.getMaxBlockTime():
block.setPendingAndCloseBlock()
self.blockCache[block.getName()] = block
def checkCompleted(self):
"""
_checkTimeout_
Loop all Open blocks and mark them as Pending if they have timed out.
"""
completedWorkflows = self.dbsUtil.getCompletedWorkflows()
for block in self.blockCache.values():
if block.status == "Open":
if block.workflow in completedWorkflows:
block.setPendingAndCloseBlock()
self.blockCache[block.getName()] = block
def addNewBlock(self, block):
"""
_addNewBlock_
Add a new block everywhere it has to go
"""
name = block.getName()
location = block.getLocation()
das = block.das
self.blockCache[name] = block
if not das in self.dasCache.keys():
self.dasCache[das] = {}
self.dasCache[das][location] = []
elif not location in self.dasCache[das].keys():
self.dasCache[das][location] = []
if name not in self.dasCache[das][location]:
self.dasCache[das][location].append(name)
return
def isBlockOpen(self, newFile, block, doTime = False):
"""
_isBlockOpen_
Check and see if a block is full
This will check on time, but that's disabled by default
The plan is to do a time check after we do everything else,
so open blocks about to time out can still get more
files put in them.
"""
if block.getMaxBlockFiles() is None or block.getMaxBlockNumEvents() is None or \
block.getMaxBlockSize() is None or block.getMaxBlockTime() is None:
return True
if block.status != 'Open':
# Then somebody has dumped this already
return False
if block.getSize() + newFile['size'] > block.getMaxBlockSize():
return False
if block.getNumEvents() + newFile['events'] > block.getMaxBlockNumEvents():
return False
if block.getNFiles() >= block.getMaxBlockFiles():
# Then we have to dump it because this file
# will put it over the limit.
return False
if block.getTime() > block.getMaxBlockTime() and doTime:
return False
return True
def getBlock(self, newFile, location, das, skipOpenCheck = False):
"""
_getBlock_
Retrieve a block is one exists and is open. If no open block is found
create and return a new one.
"""
if das in self.dasCache.keys() and location in self.dasCache[das].keys():
for blockName in self.dasCache[das][location]:
block = self.blockCache.get(blockName)
if not self.isBlockOpen(newFile = newFile, block = block) and not skipOpenCheck:
# Block isn't open anymore. Mark it as pending so that it gets
# uploaded.
block.setPendingAndCloseBlock()
self.blockCache[blockName] = block
else:
return block
# A suitable open block does not exist. Create a new one.
blockname = "%s#%s" % (newFile["datasetPath"], makeUUID())
newBlock = DBSBlock(name = blockname, location = location,
das = das, workflow = newFile["workflow"])
self.addNewBlock(block = newBlock)
return newBlock
def inputBlocks(self):
"""
_inputBlocks_
Loop through all of the "active" blocks and sort them so we can act
appropriately on them. Everything will be sorted based on the
following:
Queued - Block is already being acted on by another process. We just
ignore it.
Pending, not in DBSBuffer - Block that has been closed and needs to
be injected into DBS and also written to DBSBuffer. We'll do both.
Pending, in DBSBuffer - Block has been closed and written to
DBSBuffer. We just need to inject it into DBS.
Open, not in DBSBuffer - Newly created block that needs to be written
not DBSBuffer.
Open, in DBSBuffer - Newly created block that has already been
written to DBSBuffer. We don't have to do anything with it.
"""
myThread = threading.currentThread()
createInDBS = []
createInDBSBuffer = []
updateInDBSBuffer = []
for block in self.blockCache.values():
if block.getName() in self.queuedBlocks:
# Block is already being dealt with by another process. We'll
# ignore it here.
continue
if block.status == 'Pending':
# All pending blocks need to be injected into DBS.
createInDBS.append(block)
# If this is a new block it needs to be added to DBSBuffer
# otherwise it just needs to be updated in DBSBuffer.
if not block.inBuff:
createInDBSBuffer.append(block)
else:
updateInDBSBuffer.append(block)
if block.status == 'Open' and not block.inBuff:
# New block that needs to be added to DBSBuffer.
createInDBSBuffer.append(block)
# Build the pool if it was closed
if len(self.pool) == 0:
self.setupPool()
# First handle new and updated blocks
try:
myThread.transaction.begin()
self.dbsUtil.createBlocks(blocks = createInDBSBuffer)
self.dbsUtil.updateBlocks(blocks = updateInDBSBuffer,
dbs3UploadOnly = self.dbs3UploadOnly)
myThread.transaction.commit()
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
msg = "Unhandled exception while writing new blocks into DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for DBSBuffer: %s\n" % createInDBSBuffer)
logging.debug("Blocks for Update: %s\n" % updateInDBSBuffer)
myThread.transaction.rollback()
raise DBSUploadException(msg)
# Update block status in the block cache. Mark the blocks that we have
# added to DBSBuffer as being in DBSBuffer.
for block in createInDBSBuffer:
self.blockCache.get(block.getName()).inBuff = True
# Record new file/block associations in DBSBuffer.
try:
myThread.transaction.begin()
self.dbsUtil.setBlockFiles(binds = self.filesToUpdate)
self.filesToUpdate = []
myThread.transaction.commit()
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
msg = "Unhandled exception while setting blocks in files.\n"
msg += str(ex)
logging.error(msg)
logging.debug("Files to Update: %s\n" % self.filesToUpdate)
myThread.transaction.rollback()
raise DBSUploadException(msg)
# Finally upload blocks to DBS.
for block in createInDBS:
if len(block.files) < 1:
# What are we doing?
logging.debug("Skipping empty block")
continue
if block.getDataset() == None:
# Then we have to fix the dataset
dbsFile = block.files[0]
block.setDataset(datasetName = dbsFile['datasetPath'],
primaryType = self.primaryDatasetType,
datasetType = self.datasetType,
physicsGroup = dbsFile.get('physicsGroup', None),
prep_id = dbsFile.get('prep_id', None))
logging.debug("Found block %s in blocks" % block.getName())
block.setPhysicsGroup(group = self.physicsGroup)
encodedBlock = block.convertToDBSBlock()
logging.info("About to insert block %s" % block.getName())
self.input.put({'name': block.getName(), 'block': encodedBlock})
self.blockCount += 1
if self.produceCopy:
import json
f = open(self.copyPath, 'w')
f.write(json.dumps(encodedBlock))
f.close()
self.queuedBlocks.append(block.getName())
# And all work is in and we're done for now
return
def retrieveBlocks(self):
"""
_retrieveBlocks_
Once blocks are in DBS, we have to retrieve them and see what's
in them. What we do is get everything out of the result queue,
and then update it in DBSBuffer.
To do this, the result queue needs to pass back the blockname
"""
myThread = threading.currentThread()
blocksToClose = []
emptyCount = 0
while self.blockCount > 0:
if emptyCount > self.nTries:
# When timeoutWaiver is 0 raise error.
# It could take long time to get upload data to DBS
# if there are a lot of files are cumulated in the buffer.
# in first try but second try should be faster.
# timeoutWaiver is set as component variable - only resets when component restarted.
# The reason for that is only back log will occur when component is down
# for a long time while other component still running and feeding the data to
# dbsbuffer
if self.timeoutWaiver == 0:
msg = "Exceeded max number of waits while waiting for DBS to finish"
raise DBSUploadException(msg)
else:
self.timeoutWaiver = 0
return
try:
# Get stuff out of the queue with a ridiculously
# short wait time
blockresult = self.result.get(timeout = self.wait)
blocksToClose.append(blockresult)
self.blockCount -= 1
logging.debug("Got a block to close")
except Queue.Empty:
# This means the queue has no current results
time.sleep(2)
emptyCount += 1
continue
loadedBlocks = []
for result in blocksToClose:
# Remove from list of work being processed
self.queuedBlocks.remove(result.get('name'))
if result["success"] == "uploaded":
block = self.blockCache.get(result.get('name'))
block.status = 'InDBS'
loadedBlocks.append(block)
elif result["success"] == "check":
block = result["name"]
self.blocksToCheck.append(block)
else:
logging.error("Error found in multiprocess during process of block %s" % result.get('name'))
logging.error(result['error'])
# Continue to the next block
# Block will remain in pending status until it is transferred
try:
myThread.transaction.begin()
self.dbsUtil.updateBlocks(loadedBlocks, self.dbs3UploadOnly)
if not self.dbs3UploadOnly:
self.dbsUtil.updateFileStatus(loadedBlocks, "InDBS")
myThread.transaction.commit()
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
msg = "Unhandled exception while finished closed blocks in DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for Update: %s\n" % loadedBlocks)
myThread.transaction.rollback()
raise DBSUploadException(msg)
for block in loadedBlocks:
# Clean things up
name = block.getName()
location = block.getLocation()
das = block.das
self.dasCache[das][location].remove(name)
del self.blockCache[name]
# Clean up the pool so we don't have stuff waiting around
if len(self.pool) > 0:
self.close()
# And we're done
return
def checkBlocks(self):
"""
_checkBlocks_
Check with DBS3 if the blocks marked as check are
uploaded or not.
"""
myThread = threading.currentThread()
blocksUploaded = []
# See if there is anything to check
for block in self.blocksToCheck:
logging.debug("Checking block existence: %s" % block)
# Check in DBS if the block was really inserted
try:
result = self.dbsApi.listBlocks(block_name = block)
for blockResult in result:
if blockResult['block_name'] == block:
loadedBlock = self.blockCache.get(block)
loadedBlock.status = 'InDBS'
blocksUploaded.append(loadedBlock)
break
except Exception as ex:
exString = str(ex)
msg = "Error trying to check block %s through DBS.\n" % block
msg += exString
logging.error(msg)
logging.error(str(traceback.format_exc()))
# Update the status of those blocks that were truly inserted
try:
myThread.transaction.begin()
self.dbsUtil.updateBlocks(blocksUploaded, self.dbs3UploadOnly)
if not self.dbs3UploadOnly:
self.dbsUtil.updateFileStatus(blocksUploaded, "InDBS")
myThread.transaction.commit()
except WMException:
myThread.transaction.rollback()
raise
except Exception as ex:
msg = "Unhandled exception while finished closed blocks in DBSBuffer\n"
msg += str(ex)
logging.error(msg)
logging.debug("Blocks for Update: %s\n" % blocksUploaded)
myThread.transaction.rollback()
raise DBSUploadException(msg)
for block in blocksUploaded:
# Clean things up
name = block.getName()
location = block.getLocation()
das = block.das
self.dasCache[das][location].remove(name)
del self.blockCache[name]
# Clean the check list
self.blocksToCheck = []
# We're done
return
