def _handleChainedProcessing(self):
"""
In order to handle chained processing it's necessary to feed
output of one step/task (nomenclature ambiguous) to another.
This method creates particular mapping in a working Trivial
File Catalog (TFC).
"""
# first, create an instance of TrivialFileCatalog to override
tfc = TrivialFileCatalog()
# check the jobs input files
inputFile = "../%s/%s.root" % (self.step.data.input.inputStepName, self.step.data.input.inputOutputModule)
tfc.addMapping("direct", inputFile, inputFile, mapping_type="lfn-to-pfn")
tfc.addMapping("direct", inputFile, inputFile, mapping_type="pfn-to-lfn")
fixupFileNames(self.process)
fixupMaxEvents(self.process)
self.process.source.fileNames.setValue([inputFile])
self.process.maxEvents.input.setValue(-1)
tfcName = "override_catalog.xml"
tfcPath = os.path.join(os.getcwd(), tfcName)
print "Creating override TFC, contents below, saving into '%s'" % tfcPath
tfcStr = tfc.getXML()
print tfcStr
tfcFile = open(tfcPath, "w")
tfcFile.write(tfcStr)
tfcFile.close()
self.step.data.application.overrideCatalog = "trivialcatalog_file:" + tfcPath + "?protocol=direct"
def handleChainedProcessing(self):
"""
_handleChainedProcessing_
In order to handle chained processing it's necessary to feed
output of one step/task (nomenclature ambiguous) to another.
This method creates particular mapping in a working Trivial
File Catalog (TFC).
"""
self.logger.info("Handling chained processing job")
# first, create an instance of TrivialFileCatalog to override
tfc = TrivialFileCatalog()
# check the jobs input files
inputFile = ("../%s/%s.root" % (self.step.data.input.inputStepName,
self.step.data.input.inputOutputModule))
tfc.addMapping("direct", inputFile, inputFile, mapping_type="lfn-to-pfn")
tfc.addMapping("direct", inputFile, inputFile, mapping_type="pfn-to-lfn")
fixupFileNames(self.process)
fixupMaxEvents(self.process)
self.process.source.fileNames.setValue([inputFile])
self.process.maxEvents.input.setValue(-1)
tfcName = "override_catalog.xml"
tfcPath = os.path.join(os.getcwd(), tfcName)
self.logger.info("Creating override TFC and saving into '%s'", tfcPath)
tfcStr = tfc.getXML()
with open(tfcPath, 'w') as tfcFile:
tfcFile.write(tfcStr)
self.step.data.application.overrideCatalog = "trivialcatalog_file:" + tfcPath + "?protocol=direct"
return
def testAddMapping(self):
tfc = TrivialFileCatalog()
lfn = "some_lfn"
pfn = "some_pfn"
tfc.addMapping("direct", lfn, pfn, mapping_type = 'lfn-to-pfn')
tfc.addMapping("direct", pfn, lfn, mapping_type = 'pfn-to-lfn')
out_pfn = tfc.matchLFN("direct", lfn)
out_lfn = tfc.matchPFN("direct", pfn)
self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
def testTrivialFileCatalog(self):
"""
Run some simple tests on reading a trivialFileCatalog
"""
tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t",
"T1_US_FNAL_TrivialFileCatalog.xml")
if not os.path.exists(tfcFilename):
raise Exception("No TrivialFileCatalog found!")
tfcInstance = readTFC(tfcFilename)
self.assertEqual(type(tfcInstance), type(TrivialFileCatalog()))
# Look for similarities in each node of the TFC file
for mapping in ['lfn-to-pfn', 'pfn-to-lfn']:
for x in tfcInstance[mapping]:
self.assertEqual('path-match-expr' in x, True)
self.assertEqual('path-match' in x, True)
self.assertEqual('protocol' in x, True)
self.assertEqual('result' in x, True)
self.assertEqual('chain' in x, True)
self.assertEqual(
x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True,
'Could not find protocol %s' % (x['protocol']))
self.assertEqual(x['chain'], None,
'Invalid chain %s' % (x['chain']))
def testAddMapping(self):
tfc = TrivialFileCatalog()
lfn = "some_lfn"
pfn = "some_pfn"
tfc.addMapping("direct", lfn, pfn, mapping_type='lfn-to-pfn')
tfc.addMapping("direct", pfn, lfn, mapping_type='pfn-to-lfn')
out_pfn = tfc.matchLFN("direct", lfn)
out_lfn = tfc.matchPFN("direct", pfn)
self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
def testSaveTFC(self):
tfc = TrivialFileCatalog()
lfn = "some_lfn"
pfn = "some_pfn"
tfc.addMapping("direct", lfn, pfn, mapping_type='lfn-to-pfn')
tfc.addMapping("direct", pfn, lfn, mapping_type='pfn-to-lfn')
f = tempfile.NamedTemporaryFile("w+", delete=True) # read / write
tfcStr = str(tfc.getXML())
f.write(tfcStr)
f.flush()
f.seek(0)
tfcInstance = readTFC(f.name)
out_pfn = tfc.matchLFN("direct", lfn)
out_lfn = tfc.matchPFN("direct", pfn)
self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
f.close()
def testSaveTFC(self):
tfc = TrivialFileCatalog()
lfn = "some_lfn"
pfn = "some_pfn"
tfc.addMapping("direct", lfn, pfn, mapping_type = 'lfn-to-pfn')
tfc.addMapping("direct", pfn, lfn, mapping_type = 'pfn-to-lfn')
f = tempfile.NamedTemporaryFile("w+", delete = True) # read / write
tfcStr = str(tfc.getXML())
f.write(tfcStr)
f.flush()
f.seek(0)
tfcInstance = readTFC(f.name)
out_pfn = tfc.matchLFN("direct", lfn)
out_lfn = tfc.matchPFN("direct", pfn)
self.assertEqual(lfn, out_lfn, "Error: incorrect matching")
self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
f.close()
def testRoundTripWithChain(self):
"""
Test PFN, LFN conversion when rules are chained.
"""
tfc = TrivialFileCatalog()
match = "/+(.*)"
result = "/castor/cern.ch/cms/$1"
tfc.addMapping("direct", match, result, mapping_type="lfn-to-pfn")
match = "(.*)"
result = "$1"
tfc.addMapping("stageout",
match,
result,
chain="direct",
mapping_type="lfn-to-pfn")
in_lfn = "/this/is/a/test/lfn"
in_pfn = "/castor/cern.ch/cms" + in_lfn
out_pfn = tfc.matchLFN("stageout", in_lfn)
self.assertNotEqual(out_pfn, in_lfn)
self.assertEqual(out_pfn, in_pfn)
tfc = TrivialFileCatalog()
match = "/+castor/cern\.ch/cms/(.*)"
result = "/$1"
tfc.addMapping("direct", match, result, mapping_type="pfn-to-lfn")
match = "(.*)"
result = "$1"
tfc.addMapping("stageout",
match,
result,
chain="direct",
mapping_type="pfn-to-lfn")
out_lfn = tfc.matchPFN("stageout", in_pfn)
self.assertEqual(out_lfn, in_lfn)
def testRoundTripWithChain(self):
"""
Test PFN, LFN conversion when rules are chained.
"""
tfc = TrivialFileCatalog()
match = "/+(.*)"
result = "/castor/cern.ch/cms/$1"
tfc.addMapping("direct", match, result, mapping_type = "lfn-to-pfn")
match = "(.*)"
result = "$1"
tfc.addMapping("stageout", match, result, chain = "direct",
mapping_type = "lfn-to-pfn")
in_lfn = "/this/is/a/test/lfn"
in_pfn = "/castor/cern.ch/cms" + in_lfn
out_pfn = tfc.matchLFN("stageout", in_lfn)
self.assertNotEqual(out_pfn, in_lfn)
self.assertEqual(out_pfn, in_pfn)
tfc = TrivialFileCatalog()
match = "/+castor/cern\.ch/cms/(.*)"
result = "/$1"
tfc.addMapping("direct", match, result, mapping_type = "pfn-to-lfn")
match = "(.*)"
result = "$1"
tfc.addMapping("stageout", match, result, chain = "direct",
mapping_type = "pfn-to-lfn")
out_lfn = tfc.matchPFN("stageout", in_pfn)
self.assertEqual(out_lfn, in_lfn)
def testRoundTripWithChain(self):
"""
Test PFN, LFN conversion when rules are chained.
"""
tfc = TrivialFileCatalog()
match = "/+(.*)"
result = "/castor/cern.ch/cms/$1"
tfc.addMapping("direct", match, result, mapping_type = "lfn-to-pfn")
match = "(.*)"
result = "$1"
tfc.addMapping("stageout", match, result, chain = "direct",
mapping_type = "lfn-to-pfn")
in_lfn = "/T0/hufnagel/store/t0temp/WMAgentTier0Commissioning/A/RECO/v1dh/0000/4CF196F9-E302-E011-8517-0030487CD716.root"
in_pfn = "/castor/cern.ch/cms" + in_lfn
out_pfn = tfc.matchLFN("stageout", in_lfn)
self.assertNotEqual(out_pfn, in_lfn)
self.assertEqual(out_pfn, in_pfn)
tfc = TrivialFileCatalog()
match = "/+castor/cern\.ch/cms/(.*)"
result = "/$1"
tfc.addMapping("direct", match, result, mapping_type = "pfn-to-lfn")
match = "(.*)"
result = "$1"
tfc.addMapping("stageout", match, result, chain = "direct",
mapping_type = "pfn-to-lfn")
out_lfn = tfc.matchPFN("stageout", in_pfn)
self.assertEqual(out_lfn, in_lfn)