def setUp(self):
"""
Setup for unit tests
"""
super(DDMReqTemplateTest, self).setUp()
self.myddmReq = DDMReqTemplate('copy')
self.myddmReq['item'] = ['/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM']
def testMakeRequest(self):
expectedResult = {
'cache': None,
'group': 'DataOps',
'item': ['/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'],
'n': None,
'site': ['T2_*', 'T1_*_Disk']
}
ddmReq = DDMReqTemplate(
'copy',
item=['/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'])
result = self.myDDM.makeRequest(ddmReq)
self.assertEqual(expectedResult, result)
def testConstructor(self):
# Test construct with default values:
expectedDdmReq = {
'group': 'DataOps',
'item': ['someDatasetName'],
'site': ['T2_*', 'T1_*_Disk'],
'n': None,
'cache': None
}
self.ddmReq = DDMReqTemplate('copy', item=['someDatasetName'])
self.assertEqual(self.ddmReq, expectedDdmReq)
# Test bad API:
with self.assertRaises(ValueError):
self.ddmReq = DDMReqTemplate('coppy')
# Test bad request keys types:
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy', item='String instead of List')
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site='String instead of List')
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site=[],
group=['List instead of String'])
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site=[],
group='',
n='String instead of Int')
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site=[],
group='',
n=1,
cache=['List instead of String'])
# Test unsupported keys:
with self.assertRaises(KeyError):
self.ddmReq = DDMReqTemplate('copy', unsupported='')
def makeSubscriptions(self, workflows=[]):
"""
The common function to make the final subscriptions. It depends on the
default Data Management System configured through msConfig. Based on that
The relevant service wrapper is called.
:return: A list of results from the REST interface of the DMS in question
"""
# NOTE:
# Here is just an example construction of the function. None of the
# data structures used to visualise it is correct. To Be Updated
results = []
if self.msConfig['defaultDataManSys'] == 'DDM':
# TODO: Here to put the dryrun mode: True/False
ddm = DDM(url=self.msConfig['ddmUrl'],
logger=self.logger,
enableDataPlacement=self.msConfig['enableDataPlacement'])
ddmReqList = []
for workflow in workflows:
for output in workflow['output']:
ddmReqList.append(DDMReqTemplate('copy', item=output))
if self.msConfig['enableAggSubscr']:
results = ddm.makeAggRequests(ddmReqList, aggKey='item')
else:
for ddmReq in ddmReqList:
results.append(ddm.makeRequests(ddmReqList, aggKey='item'))
elif self.msConfig['defaultDataManSys'] == 'PhEDEx':
pass
elif self.msConfig['defaultDataManSys'] == 'Rucio':
pass
return results
def testIsEqual(self):
ddmReq1 = DDMReqTemplate('copy',
item=['DataSet1'],
site=['T2_*', 'T1_*_Disk'])
ddmReq2 = DDMReqTemplate('copy',
item=['DiffDataSet1'],
site=['T2_*', 'T1_*_Disk'])
ddmReq3 = DDMReqTemplate('copy',
item=['DataSet1'],
site=['T2_CH_CERN'])
# Test full (dictionary like) match - no key is excluded from the comparison:
self.assertFalse(ddmReq1.isEqual(ddmReq2))
self.assertFalse(ddmReq1.isEqual(ddmReq3))
self.assertFalse(ddmReq2.isEqual(ddmReq3))
# Test with 'item' key excluded from the comparison:
self.assertTrue(ddmReq1.isEqual(ddmReq2, 'item'))
self.assertFalse(ddmReq1.isEqual(ddmReq3, 'item'))
self.assertFalse(ddmReq2.isEqual(ddmReq3, 'item'))
# Test compare requests with equal keys, different APIS:
ddmReq0 = DDMReqTemplate('pollcopy',
request_id=46633,
item=['DataSet1'],
site=['T2_*', 'T1_*_Disk'])
self.assertFalse(ddmReq0.isEqual(ddmReq1))
self.assertFalse(ddmReq0.isEqual(ddmReq1, 'item'))
self.assertFalse(ddmReq0.isEqual(ddmReq2, 'item'))
self.assertFalse(ddmReq0.isEqual(ddmReq3, 'item'))
def testMakeAggRequest(self):
ddmReqList = [None] * 13
ddmReqList[0] = DDMReqTemplate(
'copy',
item=[
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
])
ddmReqList[1] = DDMReqTemplate('pollcopy', request_id=46458)
ddmReqList[2] = DDMReqTemplate(
'copy',
item=[
'/RelValSingleMuPt10Extended/CMSSW_11_1_0_pre5-110X_mcRun4_realistic_v3_2026D48noPU-v1/MINIAODSIM'
])
ddmReqList[3] = DDMReqTemplate(
'copy',
item=[
'/RelValSingleMuPt10Extended/CMSSW_11_1_0_pre5-110X_mcRun4_realistic_v3_2026D48noPU-v1/MINIAODSIM'
])
ddmReqList[4] = DDMReqTemplate(
'copy',
item=[
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
],
site=['T2_CH_CERN', 'T2_US_MIT'])
ddmReqList[5] = DDMReqTemplate('pollcopy', request_id=46614)
ddmReqList[6] = DDMReqTemplate(
'copy',
item=[
'/RelValSingleMuPt10Extended/CMSSW_11_1_0_pre5-110X_mcRun4_realistic_v3_2026D48noPU-v1/MINIAODSIM'
])
ddmReqList[7] = DDMReqTemplate(
'copy',
item=[
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
],
site=['T2_CH_CERN', 'T2_US_MIT'])
ddmReqList[8] = DDMReqTemplate('pollcopy', request_id=46627)
ddmReqList[9] = DDMReqTemplate('pollcopy', request_id=46628)
ddmReqList[10] = DDMReqTemplate('cancelcopy', request_id=46628)
ddmReqList[11] = DDMReqTemplate(
'copy',
item=[
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
],
site=['T2_CH_CERN', 'T2_US_MIT'])
ddmReqList[12] = DDMReqTemplate('pollcopy', request_id=46633)
expectedResult = [{
'item': None,
'request_id': 46633,
'site': None,
'status': None,
'user': None
}, {
'cache':
None,
'group':
'DataOps',
'item': [
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM',
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM',
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
],
'n':
None,
'site': ['T2_CH_CERN', 'T2_US_MIT']
}, {
'request_id': 46628
}, {
'item': None,
'request_id': 46628,
'site': None,
'status': None,
'user': None
}, {
'item': None,
'request_id': 46627,
'site': None,
'status': None,
'user': None
}, {
'cache':
None,
'group':
'DataOps',
'item': [
'/RelValSingleMuPt10Extended/CMSSW_11_1_0_pre5-110X_mcRun4_realistic_v3_2026D48noPU-v1/MINIAODSIM',
'/RelValSingleMuPt10Extended/CMSSW_11_1_0_pre5-110X_mcRun4_realistic_v3_2026D48noPU-v1/MINIAODSIM',
'/RelValSingleMuPt10Extended/CMSSW_11_1_0_pre5-110X_mcRun4_realistic_v3_2026D48noPU-v1/MINIAODSIM',
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
],
'n':
None,
'site': ['T2_*', 'T1_*_Disk']
}, {
'item': None,
'request_id': 46614,
'site': None,
'status': None,
'user': None
}, {
'item': None,
'request_id': 46458,
'site': None,
'status': None,
'user': None,
}]
result = self.myDDM.makeAggRequests(ddmReqList, aggKey='item')
self.assertEqual(expectedResult, result)
class DDMReqTemplateTest(EmulatedUnitTestCase):
"""
Unit tests for DDM Request templates
"""
def __init__(self, methodName='runTest'):
super(DDMReqTemplateTest, self).__init__(methodName=methodName)
def setUp(self):
"""
Setup for unit tests
"""
super(DDMReqTemplateTest, self).setUp()
self.myddmReq = DDMReqTemplate(
'copy',
item=[
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
])
def testConstructor(self):
# Test construct with default values:
expectedDdmReq = {
'group': 'DataOps',
'item': ['someDatasetName'],
'site': ['T2_*', 'T1_*_Disk'],
'n': None,
'cache': None
}
self.ddmReq = DDMReqTemplate('copy', item=['someDatasetName'])
self.assertEqual(self.ddmReq, expectedDdmReq)
# Test bad API:
with self.assertRaises(ValueError):
self.ddmReq = DDMReqTemplate('coppy')
# Test bad request keys types:
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy', item='String instead of List')
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site='String instead of List')
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site=[],
group=['List instead of String'])
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site=[],
group='',
n='String instead of Int')
with self.assertRaises(TypeError):
self.ddmReq = DDMReqTemplate('copy',
item=[],
site=[],
group='',
n=1,
cache=['List instead of String'])
# Test unsupported keys:
with self.assertRaises(KeyError):
self.ddmReq = DDMReqTemplate('copy', unsupported='')
def testStrip(self):
expectedDdmReq = {
'group':
'DataOps',
'item': [
'/LQLQToTopMuTopTau_M-1200_TuneCP5_13TeV_pythia8/RunIIFall17NanoAODv5-PU2017_12Apr2018_Nano1June2019_102X_mc2017_realistic_v7-v1/NANOAODSIM'
],
'site': ['T2_*', 'T1_*_Disk']
}
self.myddmReq.strip()
self.assertEqual(self.myddmReq, expectedDdmReq)
def testIsEqual(self):
ddmReq1 = DDMReqTemplate('copy',
item=['DataSet1'],
site=['T2_*', 'T1_*_Disk'])
ddmReq2 = DDMReqTemplate('copy',
item=['DiffDataSet1'],
site=['T2_*', 'T1_*_Disk'])
ddmReq3 = DDMReqTemplate('copy',
item=['DataSet1'],
site=['T2_CH_CERN'])
# Test full (dictionary like) match - no key is excluded from the comparison:
self.assertFalse(ddmReq1.isEqual(ddmReq2))
self.assertFalse(ddmReq1.isEqual(ddmReq3))
self.assertFalse(ddmReq2.isEqual(ddmReq3))
# Test with 'item' key excluded from the comparison:
self.assertTrue(ddmReq1.isEqual(ddmReq2, 'item'))
self.assertFalse(ddmReq1.isEqual(ddmReq3, 'item'))
self.assertFalse(ddmReq2.isEqual(ddmReq3, 'item'))
# Test compare requests with equal keys, different APIS:
ddmReq0 = DDMReqTemplate('pollcopy',
request_id=46633,
item=['DataSet1'],
site=['T2_*', 'T1_*_Disk'])
self.assertFalse(ddmReq0.isEqual(ddmReq1))
self.assertFalse(ddmReq0.isEqual(ddmReq1, 'item'))
self.assertFalse(ddmReq0.isEqual(ddmReq2, 'item'))
self.assertFalse(ddmReq0.isEqual(ddmReq3, 'item'))
def makeSubscriptions(self, workflow):
"""
The common function to make the final subscriptions. It depends on the
default Data Management System configured through msConfig. Based on that
The relevant service wrapper is called.
:return: A list of results from the REST interface of the DMS in question
"""
# NOTE:
# Here is just an example construction of the function. None of the
# data structures used to visualise it is correct. To Be Updated
if self.msConfig['defaultDataManSys'] == 'DDM':
# NOTE:
# We always aggregate per workflow here (regardless of enableAggSubscr)
# and then if we work in strides and enableAggSubscr is True then
# we will aggregate all similar subscription for all workflows
# in a single subscription - then comes the mess how to map back
# which workflow's outputs went to which transfer subscription etc.
# (TODO:)
#
# NOTE:
# Once we move to working in strides of multiple workflows at a time
# then the workflow sent to that function should not be a single one
# but an iterator of length 'stride' and then we should be doing:
# for workflow in workflows:
if isinstance(workflow, MSOutputTemplate):
ddmReqList = []
try:
if workflow['isRelVal']:
group = 'RelVal'
else:
group = 'DataOps'
for dMap in workflow['destinationOutputMap']:
try:
ddmRequest = DDMReqTemplate(
'copy',
item=dMap['datasets'],
n=workflow['numberOfCopies'],
site=dMap['destination'],
group=group)
except KeyError as ex:
# NOTE:
# If we get to here it is most probably because the 'site'
# mandatory field to the DDM request is missing (due to an
# 'ALCARECO' dataset from a Relval workflow or similar).
# Since this is expected to happen a lot, we'd better just
# log a warning and continue
msg = "Could not create DDMReq for Workflow: {}".format(
workflow['RequestName'])
msg += "Error: {}".format(ex)
self.logger.warning(msg)
continue
ddmReqList.append(ddmRequest)
except Exception as ex:
msg = "Could not create DDMReq for Workflow: {}".format(
workflow['RequestName'])
msg += "Error: {}".format(ex)
self.logger.exception(msg)
return workflow
try:
# In the message bellow we may want to put the list of datasets too
msg = "Making transfer subscriptions for %s"
self.logger.info(msg, workflow['RequestName'])
if ddmReqList:
ddmResultList = self.ddm.makeAggRequests(ddmReqList,
aggKey='item')
else:
# NOTE:
# Nothing else to be done here. We mark the document as
# done so we do not iterate through it multiple times
msg = "Skip submissions for %s. Either all data Tiers were "
msg += "excluded or there were no Output Datasets at all."
self.logger.warning(msg, workflow['RequestName'])
self.docKeyUpdate(workflow, transferStatus='done')
return workflow
except Exception as ex:
msg = "Could not make transfer subscription for Workflow: {}".format(
workflow['RequestName'])
msg += "Error: {}".format(ex)
self.logger.exception(msg)
return workflow
ddmStatusList = [
'new', 'activated', 'completed', 'rejected', 'cancelled'
]
transferIDs = []
transferStatusList = []
for ddmResult in ddmResultList:
if 'data' in ddmResult.keys():
id = deepcopy(ddmResult['data'][0]['request_id'])
status = deepcopy(ddmResult['data'][0]['status'])
transferStatusList.append({
'transferID': id,
'status': status
})
transferIDs.append(id)
if transferStatusList and all(
map(
lambda x: True
if x['status'] in ddmStatusList else False,
transferStatusList)):
self.docKeyUpdate(workflow,
transferStatus='done',
transferIDs=transferIDs)
return workflow
else:
self.docKeyUpdate(workflow, transferStatus='incomplete')
msg = "No data found in ddmResults for %s. Either dry run mode or " % workflow[
'RequestName']
msg += "broken transfer submission to DDM. "
msg += "ddmResults: \n%s" % pformat(ddmResultList)
self.logger.warning(msg)
return workflow
elif isinstance(workflow, (list, set, CommandCursor)):
ddmRequests = {}
for wflow in workflow:
wflowName = wflow['RequestName']
ddmRequests[wflowName] = DDMReqTemplate(
'copy',
item=wflow['OutputDatasets'],
n=wflow['numberOfCopies'],
site=wflow['destination'])
if self.msConfig['enableAggSubscr']:
# ddmResults = self.ddm.makeAggRequests(ddmRequests.values(), aggKey='item')
# TODO:
# Here to deal with the reverse mapping of DDM request_id to workflow
pass
else:
# for wflowName, ddmReq in ddmRequests.items():
# ddmResults.append(self.ddm.makeRequests(ddmReq))
# TODO:
# Here to deal with making request per workflow and
# reconstructing and returning the same type of object
# as the one that have been passed to the current call.
pass
else:
msg = "Unsupported type %s for workflows!\n" % type(workflow)
msg += "Skipping this call"
self.logger.error(msg)
elif self.msConfig['defaultDataManSys'] == 'PhEDEx':
pass
elif self.msConfig['defaultDataManSys'] == 'Rucio':
pass
# NOTE:
# if we are about to implement this through a pipeline we MUST not
# return the result here but the WHOLE document with updated fields
# for the transfer as it will be passed to the next function in
# the pipeline and uploaded to MongoDB
return workflow