def setUp(self):
"init test class"
self.maxDiff = None
self.msConfig = {"verbose": True,
"interval": 1 * 60,
"services": ['ruleCleaner'],
"rucioWmaAcct": 'wma_test',
"rucioAccount": 'wma_test',
'reqmgr2Url': 'https://cmsweb-testbed.cern.ch/reqmgr2',
'msOutputUrl': 'https://cmsweb-testbed.cern.ch/ms-output',
'reqmgrCacheUrl': 'https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache',
'phedexUrl': 'https://cmsweb-testbed.cern.ch/phedex/datasvc/json/prod',
'dbsUrl': 'https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader',
'rucioUrl': 'http://cms-rucio-int.cern.ch',
'rucioAuthUrl': 'https://cms-rucio-auth-int.cern.ch',
"wmstatsUrl": "https://cmsweb-testbed.cern.ch/wmstatsserver",
"logDBUrl": "https://cmsweb-testbed.cern.ch/couchdb/wmstats_logdb",
'logDBReporter': 'reqmgr2ms_ruleCleaner',
'archiveDelayHours': 8,
'enableRealMode': False}
self.creds = {"client_cert": os.getenv("X509_USER_CERT", "Unknown"),
"client_key": os.getenv("X509_USER_KEY", "Unknown")}
self.rucioConfigDict = {"rucio_host": self.msConfig['rucioUrl'],
"auth_host": self.msConfig['rucioAuthUrl'],
"auth_type": "x509",
"account": self.msConfig['rucioAccount'],
"ca_cert": False,
"timeout": 30,
"request_retries": 3,
"creds": self.creds}
self.reqStatus = ['announced', 'aborted-completed', 'rejected']
self.msRuleCleaner = MSRuleCleaner(self.msConfig)
self.msRuleCleaner.resetCounters()
self.msRuleCleaner.rucio = Rucio.Rucio(self.msConfig['rucioAccount'],
hostUrl=self.rucioConfigDict['rucio_host'],
authUrl=self.rucioConfigDict['auth_host'],
configDict=self.rucioConfigDict)
self.taskChainFile = getTestFile('data/ReqMgr/requests/Static/TaskChainRequestDump.json')
self.stepChainFile = getTestFile('data/ReqMgr/requests/Static/StepChainRequestDump.json')
self.reqRecordsFile = getTestFile('data/ReqMgr/requests/Static/BatchRequestsDump.json')
with open(self.reqRecordsFile) as fd:
self.reqRecords = json.load(fd)
with open(self.taskChainFile) as fd:
self.taskChainReq = json.load(fd)
with open(self.stepChainFile) as fd:
self.stepChainReq = json.load(fd)
super(MSRuleCleanerTest, self).setUp()
def __init__(self, config=None, logger=None):
"""
Initialize MSManager class with given configuration,
logger, ReqMgr2/ReqMgrAux/PhEDEx/Rucio objects,
and start transferor and monitoring threads.
:param config: reqmgr2ms service configuration
:param logger:
"""
self.config = config
self.logger = getMSLogger(getattr(config, 'verbose', False), logger)
self._parseConfig(config)
self.logger.info("Configuration including default values:\n%s",
self.msConfig)
self.statusTrans = {}
self.statusMon = {}
self.statusOutput = {}
self.statusRuleCleaner = {}
self.statusUnmerged = {}
# initialize transferor module
if 'transferor' in self.services:
self.msTransferor = MSTransferor(self.msConfig, logger=self.logger)
thname = 'MSTransferor'
self.transfThread = start_new_thread(
thname, daemon, (self.transferor, 'assigned',
self.msConfig['interval'], self.logger))
self.logger.info("### Running %s thread %s", thname,
self.transfThread.running())
# initialize monitoring module
if 'monitor' in self.services:
self.msMonitor = MSMonitor(self.msConfig, logger=self.logger)
thname = 'MSMonitor'
self.monitThread = start_new_thread(
thname, daemon, (self.monitor, 'staging',
self.msConfig['interval'], self.logger))
self.logger.info("+++ Running %s thread %s", thname,
self.monitThread.running())
# initialize output module
if 'output' in self.services:
reqStatus = ['closed-out', 'announced']
# thread safe cache to keep the last X requests processed in MSOutput
requestNamesCached = deque(
maxlen=self.msConfig.get("cacheRequestSize", 10000))
thname = 'MSOutputConsumer'
self.msOutputConsumer = MSOutput(self.msConfig,
mode=thname,
reqCache=requestNamesCached,
logger=self.logger)
# set the consumer to run twice faster than the producer
consumerInterval = self.msConfig['interval'] // 2
self.outputConsumerThread = start_new_thread(
thname, daemon, (self.outputConsumer, reqStatus,
consumerInterval, self.logger))
self.logger.info("=== Running %s thread %s", thname,
self.outputConsumerThread.running())
thname = 'MSOutputProducer'
self.msOutputProducer = MSOutput(self.msConfig,
mode=thname,
reqCache=requestNamesCached,
logger=self.logger)
self.outputProducerThread = start_new_thread(
thname, daemon, (self.outputProducer, reqStatus,
self.msConfig['interval'], self.logger))
self.logger.info("=== Running %s thread %s", thname,
self.outputProducerThread.running())
# initialize rule cleaner module
if 'ruleCleaner' in self.services:
reqStatus = ['announced', 'aborted-completed', 'rejected']
self.msRuleCleaner = MSRuleCleaner(self.msConfig,
logger=self.logger)
thname = 'MSRuleCleaner'
self.ruleCleanerThread = start_new_thread(
thname, daemon, (self.ruleCleaner, reqStatus,
self.msConfig['interval'], self.logger))
self.logger.info("--- Running %s thread %s", thname,
self.ruleCleanerThread.running())
# initialize unmerged module
if 'unmerged' in self.services:
self.msUnmerged = MSUnmerged(self.msConfig, logger=self.logger)
thname = 'MSUnmerged'
self.unmergedThread = start_new_thread(
thname, daemonOpt,
(self.unmerged, self.msConfig['interval'], self.logger))
self.logger.info("--- Running %s thread %s", thname,
self.unmergedThread.running())
class MSManager(object):
"""
Entry point for the MicroServices.
This class manages both transferor and monitoring services.
"""
def __init__(self, config=None, logger=None):
"""
Initialize MSManager class with given configuration,
logger, ReqMgr2/ReqMgrAux/PhEDEx/Rucio objects,
and start transferor and monitoring threads.
:param config: reqmgr2ms service configuration
:param logger:
"""
self.config = config
self.logger = getMSLogger(getattr(config, 'verbose', False), logger)
self._parseConfig(config)
self.logger.info("Configuration including default values:\n%s",
self.msConfig)
self.statusTrans = {}
self.statusMon = {}
self.statusOutput = {}
self.statusRuleCleaner = {}
self.statusUnmerged = {}
# initialize transferor module
if 'transferor' in self.services:
self.msTransferor = MSTransferor(self.msConfig, logger=self.logger)
thname = 'MSTransferor'
self.transfThread = start_new_thread(
thname, daemon, (self.transferor, 'assigned',
self.msConfig['interval'], self.logger))
self.logger.info("### Running %s thread %s", thname,
self.transfThread.running())
# initialize monitoring module
if 'monitor' in self.services:
self.msMonitor = MSMonitor(self.msConfig, logger=self.logger)
thname = 'MSMonitor'
self.monitThread = start_new_thread(
thname, daemon, (self.monitor, 'staging',
self.msConfig['interval'], self.logger))
self.logger.info("+++ Running %s thread %s", thname,
self.monitThread.running())
# initialize output module
if 'output' in self.services:
reqStatus = ['closed-out', 'announced']
# thread safe cache to keep the last X requests processed in MSOutput
requestNamesCached = deque(
maxlen=self.msConfig.get("cacheRequestSize", 10000))
thname = 'MSOutputConsumer'
self.msOutputConsumer = MSOutput(self.msConfig,
mode=thname,
reqCache=requestNamesCached,
logger=self.logger)
# set the consumer to run twice faster than the producer
consumerInterval = self.msConfig['interval'] // 2
self.outputConsumerThread = start_new_thread(
thname, daemon, (self.outputConsumer, reqStatus,
consumerInterval, self.logger))
self.logger.info("=== Running %s thread %s", thname,
self.outputConsumerThread.running())
thname = 'MSOutputProducer'
self.msOutputProducer = MSOutput(self.msConfig,
mode=thname,
reqCache=requestNamesCached,
logger=self.logger)
self.outputProducerThread = start_new_thread(
thname, daemon, (self.outputProducer, reqStatus,
self.msConfig['interval'], self.logger))
self.logger.info("=== Running %s thread %s", thname,
self.outputProducerThread.running())
# initialize rule cleaner module
if 'ruleCleaner' in self.services:
reqStatus = ['announced', 'aborted-completed', 'rejected']
self.msRuleCleaner = MSRuleCleaner(self.msConfig,
logger=self.logger)
thname = 'MSRuleCleaner'
self.ruleCleanerThread = start_new_thread(
thname, daemon, (self.ruleCleaner, reqStatus,
self.msConfig['interval'], self.logger))
self.logger.info("--- Running %s thread %s", thname,
self.ruleCleanerThread.running())
# initialize unmerged module
if 'unmerged' in self.services:
self.msUnmerged = MSUnmerged(self.msConfig, logger=self.logger)
thname = 'MSUnmerged'
self.unmergedThread = start_new_thread(
thname, daemonOpt,
(self.unmerged, self.msConfig['interval'], self.logger))
self.logger.info("--- Running %s thread %s", thname,
self.unmergedThread.running())
def _parseConfig(self, config):
"""
__parseConfig_
Parse the MicroService configuration and set any default values.
:param config: config as defined in the deployment
"""
self.logger.info("Using the following MicroServices config: %s",
config.dictionary_())
self.services = getattr(config, 'services', [])
self.msConfig = {}
self.msConfig.update(config.dictionary_())
self.msConfig['reqmgrCacheUrl'] = self.msConfig['reqmgr2Url'].replace(
'reqmgr2', 'couchdb/reqmgr_workload_cache')
def transferor(self, reqStatus):
"""
MSManager transferor function.
It performs Unified logic for data subscription and
transfers requests from assigned to staging/staged state of ReqMgr2.
For references see
https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Transferor
"""
startTime = datetime.utcnow()
self.logger.info("Starting the transferor thread...")
res = self.msTransferor.execute(reqStatus)
endTime = datetime.utcnow()
self.updateTimeUTC(res, startTime, endTime)
self.logger.info("Total transferor execution time: %.2f secs",
res['execution_time'])
self.statusTrans = res
def monitor(self, reqStatus):
"""
MSManager monitoring function.
It performs transfer requests from staging to staged state of ReqMgr2.
For references see
https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Transferor
"""
startTime = datetime.utcnow()
self.logger.info("Starting the monitor thread...")
res = self.msMonitor.execute(reqStatus)
endTime = datetime.utcnow()
self.updateTimeUTC(res, startTime, endTime)
self.logger.info("Total monitor execution time: %d secs",
res['execution_time'])
self.statusMon = res
def outputConsumer(self, reqStatus):
"""
MSManager Output Data Placement function.
It subscribes the output datasets to the Data Management System.
For references see
https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Output
reqStatus: Status of requests to work on
"""
startTime = datetime.utcnow()
self.logger.info("Starting the outputConsumer thread...")
res = self.msOutputConsumer.execute(reqStatus)
endTime = datetime.utcnow()
self.updateTimeUTC(res, startTime, endTime)
self.logger.info("Total outputConsumer execution time: %d secs",
res['execution_time'])
self.statusOutput = res
def outputProducer(self, reqStatus):
"""
MSManager MongoDB Uploader function.
It uploads the documents describing a workflow output Data subscription
into MongoDb. For references see
https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Output
reqStatus: Status of requests to work on
"""
startTime = datetime.utcnow()
self.logger.info("Starting the outputProducer thread...")
res = self.msOutputProducer.execute(reqStatus)
endTime = datetime.utcnow()
self.updateTimeUTC(res, startTime, endTime)
self.logger.info("Total outputProducer execution time: %d secs",
res['execution_time'])
self.statusOutput = res
def ruleCleaner(self, reqStatus):
"""
MSManager ruleCleaner function.
It cleans the block level Rucio rules created by WMAgent and
performs request status transition from ['announced', 'aborted-completed', 'rejected'] to
'{normal, aborted, rejected}-archived' state of ReqMgr2.
For references see
https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-RuleCleaner
"""
startTime = datetime.utcnow()
self.logger.info("Starting the ruleCleaner thread...")
res = self.msRuleCleaner.execute(reqStatus)
endTime = datetime.utcnow()
self.updateTimeUTC(res, startTime, endTime)
self.logger.info("Total ruleCleaner execution time: %d secs",
res['execution_time'])
self.statusRuleCleaner = res
def unmerged(self, *args, **kwargs):
"""
MSManager unmerged function.
It cleans the Unmerged area of the CMS LFN Namespace
For references see
https://github.com/dmwm/WMCore/wiki/ReqMgr2-MicroService-Unmerged
"""
startTime = datetime.utcnow()
self.logger.info("Starting the unmerged thread...")
res = self.msUnmerged.execute()
endTime = datetime.utcnow()
self.updateTimeUTC(res, startTime, endTime)
self.logger.info("Total Unmerged execution time: %d secs",
res['execution_time'])
self.statusUnmerged = res
def stop(self):
"Stop MSManager"
status = None
# stop MSMonitor thread
if 'monitor' in self.services and hasattr(self, 'monitThread'):
self.monitThread.stop()
status = self.monitThread.running()
# stop MSTransferor thread
if 'transferor' in self.services and hasattr(self, 'transfThread'):
self.transfThread.stop() # stop checkStatus thread
status = self.transfThread.running()
# stop MSOutput threads
if 'output' in self.services and hasattr(self, 'outputConsumerThread'):
self.outputConsumerThread.stop()
status = self.outputConsumerThread.running()
if 'output' in self.services and hasattr(self, 'outputProducerThread'):
self.outputProducerThread.stop()
status = self.outputProducerThread.running()
# stop MSRuleCleaner thread
if 'ruleCleaner' in self.services and hasattr(self,
'ruleCleanerThread'):
self.ruleCleanerThread.stop()
status = self.ruleCleanerThread.running()
return status
def info(self, reqName=None):
"""
Return transfer information for a given request
:param reqName: request name
:return: data transfer information for this request
"""
data = {"request": reqName, "transferDoc": None}
if reqName:
# obtain the transfer information for a given request records from couchdb for given request
if 'monitor' in self.services:
transferDoc = self.msMonitor.reqmgrAux.getTransferInfo(reqName)
elif 'transferor' in self.services:
transferDoc = self.msTransferor.reqmgrAux.getTransferInfo(
reqName)
elif 'output' in self.services:
transferDoc = self.msOutputProducer.getTransferInfo(reqName)
if transferDoc:
# it's always a single document in Couch
data['transferDoc'] = transferDoc[0]
return data
def delete(self, request):
"Delete request in backend"
pass
def status(self, detail):
"""
Return the current status of a MicroService and a summary
of its last execution activity.
:param detail: boolean used to retrieve some extra information
regarding the service
:return: a dictionary
"""
data = {"status": "OK"}
if detail and 'transferor' in self.services:
data.update(self.statusTrans)
elif detail and 'monitor' in self.services:
data.update(self.statusMon)
elif detail and 'output' in self.services:
data.update(self.statusOutput)
elif detail and 'ruleCleaner' in self.services:
data.update(self.statusRuleCleaner)
return data
def updateTimeUTC(self, reportDict, startT, endT):
"""
Given a report summary dictionary and start/end time, update
the report with human readable timing information
:param reportDict: summary dictionary
:param startT: epoch start time for a given service
:param endT: epoch end time for a given service
"""
reportDict['start_time'] = startT.strftime("%a, %d %b %Y %H:%M:%S UTC")
reportDict['end_time'] = endT.strftime("%a, %d %b %Y %H:%M:%S UTC")
reportDict['execution_time'] = (endT - startT).total_seconds()
class MSRuleCleanerTest(unittest.TestCase):
"Unit test for MSruleCleaner module"
def setUp(self):
"init test class"
self.maxDiff = None
self.msConfig = {
"verbose": True,
"interval": 1 * 60,
"services": ['ruleCleaner'],
"rucioWmaAcct": 'wma_test',
"rucioAccount": 'wma_test',
'reqmgr2Url': 'https://cmsweb-testbed.cern.ch/reqmgr2',
'msOutputUrl': 'https://cmsweb-testbed.cern.ch/ms-output',
'reqmgrCacheUrl':
'https://cmsweb-testbed.cern.ch/couchdb/reqmgr_workload_cache',
'phedexUrl':
'https://cmsweb-testbed.cern.ch/phedex/datasvc/json/prod',
'dbsUrl':
'https://cmsweb-testbed.cern.ch/dbs/int/global/DBSReader',
'rucioUrl': 'http://cmsrucio-int.cern.ch',
'rucioAuthUrl': 'https://cmsrucio-auth-int.cern.ch',
"wmstatsUrl": "https://cmsweb-testbed.cern.ch/wmstatsserver",
"logDBUrl": "https://cmsweb-testbed.cern.ch/couchdb/wmstats_logdb",
'logDBReporter': 'reqmgr2ms_ruleCleaner',
'archiveDelayHours': 8,
'enableRealMode': False
}
self.creds = {
"client_cert": os.getenv("X509_USER_CERT", "Unknown"),
"client_key": os.getenv("X509_USER_KEY", "Unknown")
}
self.rucioConfigDict = {
"rucio_host": self.msConfig['rucioUrl'],
"auth_host": self.msConfig['rucioAuthUrl'],
"auth_type": "x509",
"account": self.msConfig['rucioAccount'],
"ca_cert": False,
"timeout": 30,
"request_retries": 3,
"creds": self.creds
}
self.reqStatus = ['announced', 'aborted-completed', 'rejected']
self.msRuleCleaner = MSRuleCleaner(self.msConfig)
self.msRuleCleaner.resetCounters()
self.msRuleCleaner.rucio = Rucio.Rucio(
self.msConfig['rucioAccount'],
hostUrl=self.rucioConfigDict['rucio_host'],
authUrl=self.rucioConfigDict['auth_host'],
configDict=self.rucioConfigDict)
self.taskChainFile = getTestFile(
'data/ReqMgr/requests/Static/TaskChainRequestDump.json')
self.stepChainFile = getTestFile(
'data/ReqMgr/requests/Static/StepChainRequestDump.json')
self.reqRecordsFile = getTestFile(
'data/ReqMgr/requests/Static/BatchRequestsDump.json')
with open(self.reqRecordsFile) as fd:
self.reqRecords = json.load(fd)
with open(self.taskChainFile) as fd:
self.taskChainReq = json.load(fd)
with open(self.stepChainFile) as fd:
self.stepChainReq = json.load(fd)
super(MSRuleCleanerTest, self).setUp()
def testPipelineAgentBlock(self):
# Test plineAgentBlock:
wflow = MSRuleCleanerWflow(self.taskChainReq)
self.msRuleCleaner.plineAgentBlock.run(wflow)
expectedWflow = {
'CleanupStatus': {
'plineAgentBlock': True
},
'DataPileup': [],
'ForceArchive':
False,
'IncludeParents':
False,
'InputDataset':
u'/JetHT/Run2012C-v1/RAW',
'IsArchivalDelayExpired':
False,
'IsClean':
False,
'IsLogDBClean':
False,
'MCPileup': [],
'OutputDatasets': [
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO',
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO',
u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'
],
'ParentDataset': [],
'ParentageResolved':
True,
'PlineMarkers': ['plineAgentBlock'],
'RequestName':
u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891',
'RequestStatus':
u'announced',
'RequestTransition': [{
u'DN': u'',
u'Status': u'new',
u'UpdateTime': 1606723304
}, {
u'DN': u'',
u'Status': u'assignment-approved',
u'UpdateTime': 1606723305
}, {
u'DN': u'',
u'Status': u'assigned',
u'UpdateTime': 1606723306
}, {
u'DN': u'',
u'Status': u'staging',
u'UpdateTime': 1606723461
}, {
u'DN': u'',
u'Status': u'staged',
u'UpdateTime': 1606723590
}, {
u'DN': u'',
u'Status': u'acquired',
u'UpdateTime': 1606723968
}, {
u'DN': u'',
u'Status': u'running-open',
u'UpdateTime': 1606724572
}, {
u'DN': u'',
u'Status': u'running-closed',
u'UpdateTime': 1606724573
}, {
u'DN': u'',
u'Status': u'completed',
u'UpdateTime': 1607018413
}, {
u'DN': u'',
u'Status': u'closed-out',
u'UpdateTime': 1607347706
}, {
u'DN': u'',
u'Status': u'announced',
u'UpdateTime': 1607359514
}],
'RequestType':
u'TaskChain',
'RulesToClean': {
'plineAgentBlock': []
},
'TargetStatus':
None,
'TransferDone':
False,
'TransferTape':
False
}
self.assertDictEqual(wflow, expectedWflow)
def testPipelineAgentCont(self):
# Test plineAgentCont
wflow = MSRuleCleanerWflow(self.taskChainReq)
self.msRuleCleaner.plineAgentCont.run(wflow)
expectedWflow = {
'CleanupStatus': {
'plineAgentCont': True
},
'DataPileup': [],
'ForceArchive':
False,
'IncludeParents':
False,
'InputDataset':
u'/JetHT/Run2012C-v1/RAW',
'IsArchivalDelayExpired':
False,
'IsClean':
False,
'IsLogDBClean':
False,
'MCPileup': [],
'OutputDatasets': [
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO',
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO',
u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'
],
'ParentDataset': [],
'ParentageResolved':
True,
'PlineMarkers': ['plineAgentCont'],
'RequestName':
u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891',
'RequestStatus':
u'announced',
'RequestTransition': [{
u'DN': u'',
u'Status': u'new',
u'UpdateTime': 1606723304
}, {
u'DN': u'',
u'Status': u'assignment-approved',
u'UpdateTime': 1606723305
}, {
u'DN': u'',
u'Status': u'assigned',
u'UpdateTime': 1606723306
}, {
u'DN': u'',
u'Status': u'staging',
u'UpdateTime': 1606723461
}, {
u'DN': u'',
u'Status': u'staged',
u'UpdateTime': 1606723590
}, {
u'DN': u'',
u'Status': u'acquired',
u'UpdateTime': 1606723968
}, {
u'DN': u'',
u'Status': u'running-open',
u'UpdateTime': 1606724572
}, {
u'DN': u'',
u'Status': u'running-closed',
u'UpdateTime': 1606724573
}, {
u'DN': u'',
u'Status': u'completed',
u'UpdateTime': 1607018413
}, {
u'DN': u'',
u'Status': u'closed-out',
u'UpdateTime': 1607347706
}, {
u'DN': u'',
u'Status': u'announced',
u'UpdateTime': 1607359514
}],
'RequestType':
u'TaskChain',
'RulesToClean': {
'plineAgentCont': []
},
'TargetStatus':
None,
'TransferDone':
False,
'TransferTape':
False
}
self.assertDictEqual(wflow, expectedWflow)
def testPipelineMSTrBlock(self):
# Test plineAgentCont
wflow = MSRuleCleanerWflow(self.taskChainReq)
self.msRuleCleaner.plineMSTrBlock.run(wflow)
expectedWflow = {
'CleanupStatus': {
'plineMSTrBlock': True
},
'DataPileup': [],
'ForceArchive':
False,
'IncludeParents':
False,
'InputDataset':
u'/JetHT/Run2012C-v1/RAW',
'IsArchivalDelayExpired':
False,
'IsClean':
False,
'IsLogDBClean':
False,
'MCPileup': [],
'OutputDatasets': [
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO',
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO',
u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'
],
'ParentDataset': [],
'ParentageResolved':
True,
'PlineMarkers': ['plineMSTrBlock'],
'RequestName':
u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891',
'RequestStatus':
u'announced',
'RequestTransition': [{
u'DN': u'',
u'Status': u'new',
u'UpdateTime': 1606723304
}, {
u'DN': u'',
u'Status': u'assignment-approved',
u'UpdateTime': 1606723305
}, {
u'DN': u'',
u'Status': u'assigned',
u'UpdateTime': 1606723306
}, {
u'DN': u'',
u'Status': u'staging',
u'UpdateTime': 1606723461
}, {
u'DN': u'',
u'Status': u'staged',
u'UpdateTime': 1606723590
}, {
u'DN': u'',
u'Status': u'acquired',
u'UpdateTime': 1606723968
}, {
u'DN': u'',
u'Status': u'running-open',
u'UpdateTime': 1606724572
}, {
u'DN': u'',
u'Status': u'running-closed',
u'UpdateTime': 1606724573
}, {
u'DN': u'',
u'Status': u'completed',
u'UpdateTime': 1607018413
}, {
u'DN': u'',
u'Status': u'closed-out',
u'UpdateTime': 1607347706
}, {
u'DN': u'',
u'Status': u'announced',
u'UpdateTime': 1607359514
}],
'RequestType':
u'TaskChain',
'RulesToClean': {
'plineMSTrBlock': []
},
'TargetStatus':
None,
'TransferDone':
False,
'TransferTape':
False
}
self.assertDictEqual(wflow, expectedWflow)
def testPipelineMSTrCont(self):
# Test plineAgentCont
wflow = MSRuleCleanerWflow(self.taskChainReq)
self.msRuleCleaner.plineMSTrCont.run(wflow)
expectedWflow = {
'CleanupStatus': {
'plineMSTrCont': True
},
'DataPileup': [],
'ForceArchive':
False,
'IncludeParents':
False,
'InputDataset':
u'/JetHT/Run2012C-v1/RAW',
'IsArchivalDelayExpired':
False,
'IsClean':
False,
'IsLogDBClean':
False,
'MCPileup': [],
'OutputDatasets': [
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO',
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO',
u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'
],
'ParentDataset': [],
'ParentageResolved':
True,
'PlineMarkers': ['plineMSTrCont'],
'RequestName':
u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891',
'RequestStatus':
u'announced',
'RequestTransition': [{
u'DN': u'',
u'Status': u'new',
u'UpdateTime': 1606723304
}, {
u'DN': u'',
u'Status': u'assignment-approved',
u'UpdateTime': 1606723305
}, {
u'DN': u'',
u'Status': u'assigned',
u'UpdateTime': 1606723306
}, {
u'DN': u'',
u'Status': u'staging',
u'UpdateTime': 1606723461
}, {
u'DN': u'',
u'Status': u'staged',
u'UpdateTime': 1606723590
}, {
u'DN': u'',
u'Status': u'acquired',
u'UpdateTime': 1606723968
}, {
u'DN': u'',
u'Status': u'running-open',
u'UpdateTime': 1606724572
}, {
u'DN': u'',
u'Status': u'running-closed',
u'UpdateTime': 1606724573
}, {
u'DN': u'',
u'Status': u'completed',
u'UpdateTime': 1607018413
}, {
u'DN': u'',
u'Status': u'closed-out',
u'UpdateTime': 1607347706
}, {
u'DN': u'',
u'Status': u'announced',
u'UpdateTime': 1607359514
}],
'RequestType':
u'TaskChain',
'RulesToClean': {
'plineMSTrCont': []
},
'TargetStatus':
None,
'TransferDone':
False,
'TransferTape':
False
}
self.assertDictEqual(wflow, expectedWflow)
def testPipelineArchive(self):
# Test plineAgentCont
wflow = MSRuleCleanerWflow(self.taskChainReq)
# Try archival of a skipped workflow:
with self.assertRaises(MSRuleCleanerArchivalSkip):
self.msRuleCleaner.plineArchive.run(wflow)
self.msRuleCleaner.plineAgentBlock.run(wflow)
self.msRuleCleaner.plineAgentCont.run(wflow)
# Try archival of a cleaned workflow:
# NOTE: We should always expect an MSRuleCleanerArchivalSkip exception
# here because the 'enableRealRunMode' flag is set to False
with self.assertRaises(MSRuleCleanerArchivalSkip):
self.msRuleCleaner.plineArchive.run(wflow)
expectedWflow = {
'CleanupStatus': {
'plineAgentBlock': True,
'plineAgentCont': True
},
'DataPileup': [],
'ForceArchive':
False,
'IncludeParents':
False,
'InputDataset':
u'/JetHT/Run2012C-v1/RAW',
'IsArchivalDelayExpired':
True,
'IsClean':
True,
'IsLogDBClean':
True,
'MCPileup': [],
'OutputDatasets': [
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/RECO',
u'/JetHT/CMSSW_7_2_0-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/DQMIO',
u'/JetHT/CMSSW_7_2_0-SiStripCalMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-SiStripCalZeroBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO',
u'/JetHT/CMSSW_7_2_0-TkAlMinBias-RECODreHLT_TaskChain_LumiMask_multiRun_HG2011_Val_Todor_v1-v11/ALCARECO'
],
'ParentDataset': [],
'ParentageResolved':
True,
'PlineMarkers': [
'plineArchive', 'plineAgentBlock', 'plineAgentCont',
'plineArchive'
],
'RequestName':
u'TaskChain_LumiMask_multiRun_HG2011_Val_201029_112735_5891',
'RequestStatus':
u'announced',
'RequestTransition': [{
u'DN': u'',
u'Status': u'new',
u'UpdateTime': 1606723304
}, {
u'DN': u'',
u'Status': u'assignment-approved',
u'UpdateTime': 1606723305
}, {
u'DN': u'',
u'Status': u'assigned',
u'UpdateTime': 1606723306
}, {
u'DN': u'',
u'Status': u'staging',
u'UpdateTime': 1606723461
}, {
u'DN': u'',
u'Status': u'staged',
u'UpdateTime': 1606723590
}, {
u'DN': u'',
u'Status': u'acquired',
u'UpdateTime': 1606723968
}, {
u'DN': u'',
u'Status': u'running-open',
u'UpdateTime': 1606724572
}, {
u'DN': u'',
u'Status': u'running-closed',
u'UpdateTime': 1606724573
}, {
u'DN': u'',
u'Status': u'completed',
u'UpdateTime': 1607018413
}, {
u'DN': u'',
u'Status': u'closed-out',
u'UpdateTime': 1607347706
}, {
u'DN': u'',
u'Status': u'announced',
u'UpdateTime': 1607359514
}],
'RequestType':
u'TaskChain',
'RulesToClean': {
'plineAgentBlock': [],
'plineAgentCont': []
},
'TargetStatus':
'normal-archived',
'TransferDone':
False,
'TransferTape':
False
}
self.assertDictEqual(wflow, expectedWflow)
# Try archival of an uncleaned workflow
wflow['CleanupStatus']['plineAgentBlock'] = False
with self.assertRaises(MSRuleCleanerArchivalSkip):
self.msRuleCleaner.plineArchive.run(wflow)
def testRunning(self):
result = self.msRuleCleaner._execute(self.reqRecords)
self.assertEqual(result, (3, 2, 0, 0))
def testCheckClean(self):
# NOTE: All of the bellow checks are well visualized at:
# https://github.com/dmwm/WMCore/pull/10023#discussion_r520070925
# 1. MaskList shorter than FlagList
wflowFlags = {
'CleanupStatus': {
'plineAgentBlock': True,
'plineAgentCont': True,
'plineMStrCont': False
},
'PlineMarkers': ['plineAgentBlock', 'plineAgentCont']
}
self.assertTrue(self.msRuleCleaner._checkClean(wflowFlags))
wflowFlags = {
'CleanupStatus': {
'plineAgentBlock': False,
'plineAgentCont': True,
'plineMStrCont': True
},
'PlineMarkers': ['plineAgentBlock', 'plineAgentCont']
}
self.assertFalse(self.msRuleCleaner._checkClean(wflowFlags))
# 2. MaskList Empty
wflowFlags = {
'CleanupStatus': {
'plineAgentBlock': True,
'plineAgentCont': True
},
'PlineMarkers': []
}
self.assertFalse(self.msRuleCleaner._checkClean(wflowFlags))
# 3. MaskList longer than FlagList
wflowFlags = {
'CleanupStatus': {
'plineAgentBlock': True,
'plineAgentCont': True
},
'PlineMarkers': [
'plineAgentBlock', 'plineAgentCont', 'plineMStrCont',
'plineArchive'
]
}
self.assertTrue(self.msRuleCleaner._checkClean(wflowFlags))
wflowFlags = {
'CleanupStatus': {
'plineAgentBlock': True,
'plineAgentCont': False
},
'PlineMarkers': [
'plineAgentBlock', 'plineAgentCont', 'plineMStrCont',
'plineArchive'
]
}
self.assertFalse(self.msRuleCleaner._checkClean(wflowFlags))
# 4. FlagList Empty
wflowFlags = {
'CleanupStatus': {},
'PlineMarkers': ['plineAgentBlock', 'plineAgentCont']
}
self.assertFalse(self.msRuleCleaner._checkClean(wflowFlags))