def _handleChainedProcessing(self): """ In order to handle chained processing it's necessary to feed output of one step/task (nomenclature ambiguous) to another. This method creates particular mapping in a working Trivial File Catalog (TFC). """ # first, create an instance of TrivialFileCatalog to override tfc = TrivialFileCatalog() # check the jobs input files inputFile = "../%s/%s.root" % (self.step.data.input.inputStepName, self.step.data.input.inputOutputModule) tfc.addMapping("direct", inputFile, inputFile, mapping_type="lfn-to-pfn") tfc.addMapping("direct", inputFile, inputFile, mapping_type="pfn-to-lfn") fixupFileNames(self.process) fixupMaxEvents(self.process) self.process.source.fileNames.setValue([inputFile]) self.process.maxEvents.input.setValue(-1) tfcName = "override_catalog.xml" tfcPath = os.path.join(os.getcwd(), tfcName) print "Creating override TFC, contents below, saving into '%s'" % tfcPath tfcStr = tfc.getXML() print tfcStr tfcFile = open(tfcPath, "w") tfcFile.write(tfcStr) tfcFile.close() self.step.data.application.overrideCatalog = "trivialcatalog_file:" + tfcPath + "?protocol=direct"
def handleChainedProcessing(self): """ _handleChainedProcessing_ In order to handle chained processing it's necessary to feed output of one step/task (nomenclature ambiguous) to another. This method creates particular mapping in a working Trivial File Catalog (TFC). """ self.logger.info("Handling chained processing job") # first, create an instance of TrivialFileCatalog to override tfc = TrivialFileCatalog() # check the jobs input files inputFile = ("../%s/%s.root" % (self.step.data.input.inputStepName, self.step.data.input.inputOutputModule)) tfc.addMapping("direct", inputFile, inputFile, mapping_type="lfn-to-pfn") tfc.addMapping("direct", inputFile, inputFile, mapping_type="pfn-to-lfn") fixupFileNames(self.process) fixupMaxEvents(self.process) self.process.source.fileNames.setValue([inputFile]) self.process.maxEvents.input.setValue(-1) tfcName = "override_catalog.xml" tfcPath = os.path.join(os.getcwd(), tfcName) self.logger.info("Creating override TFC and saving into '%s'", tfcPath) tfcStr = tfc.getXML() with open(tfcPath, 'w') as tfcFile: tfcFile.write(tfcStr) self.step.data.application.overrideCatalog = "trivialcatalog_file:" + tfcPath + "?protocol=direct" return
def testAddMapping(self): tfc = TrivialFileCatalog() lfn = "some_lfn" pfn = "some_pfn" tfc.addMapping("direct", lfn, pfn, mapping_type = 'lfn-to-pfn') tfc.addMapping("direct", pfn, lfn, mapping_type = 'pfn-to-lfn') out_pfn = tfc.matchLFN("direct", lfn) out_lfn = tfc.matchPFN("direct", pfn) self.assertEqual(lfn, out_lfn, "Error: incorrect matching") self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
def testTrivialFileCatalog(self): """ Run some simple tests on reading a trivialFileCatalog """ tfcFilename = os.path.join(getTestBase(), "WMCore_t/Storage_t", "T1_US_FNAL_TrivialFileCatalog.xml") if not os.path.exists(tfcFilename): raise Exception("No TrivialFileCatalog found!") tfcInstance = readTFC(tfcFilename) self.assertEqual(type(tfcInstance), type(TrivialFileCatalog())) # Look for similarities in each node of the TFC file for mapping in ['lfn-to-pfn', 'pfn-to-lfn']: for x in tfcInstance[mapping]: self.assertEqual('path-match-expr' in x, True) self.assertEqual('path-match' in x, True) self.assertEqual('protocol' in x, True) self.assertEqual('result' in x, True) self.assertEqual('chain' in x, True) self.assertEqual( x['protocol'] in ['direct', 'dcap', 'srm', 'srmv2'], True, 'Could not find protocol %s' % (x['protocol'])) self.assertEqual(x['chain'], None, 'Invalid chain %s' % (x['chain']))
def testAddMapping(self): tfc = TrivialFileCatalog() lfn = "some_lfn" pfn = "some_pfn" tfc.addMapping("direct", lfn, pfn, mapping_type='lfn-to-pfn') tfc.addMapping("direct", pfn, lfn, mapping_type='pfn-to-lfn') out_pfn = tfc.matchLFN("direct", lfn) out_lfn = tfc.matchPFN("direct", pfn) self.assertEqual(lfn, out_lfn, "Error: incorrect matching") self.assertEqual(pfn, out_pfn, "Error: incorrect matching")
def testSaveTFC(self): tfc = TrivialFileCatalog() lfn = "some_lfn" pfn = "some_pfn" tfc.addMapping("direct", lfn, pfn, mapping_type='lfn-to-pfn') tfc.addMapping("direct", pfn, lfn, mapping_type='pfn-to-lfn') f = tempfile.NamedTemporaryFile("w+", delete=True) # read / write tfcStr = str(tfc.getXML()) f.write(tfcStr) f.flush() f.seek(0) tfcInstance = readTFC(f.name) out_pfn = tfc.matchLFN("direct", lfn) out_lfn = tfc.matchPFN("direct", pfn) self.assertEqual(lfn, out_lfn, "Error: incorrect matching") self.assertEqual(pfn, out_pfn, "Error: incorrect matching") f.close()
def testSaveTFC(self): tfc = TrivialFileCatalog() lfn = "some_lfn" pfn = "some_pfn" tfc.addMapping("direct", lfn, pfn, mapping_type = 'lfn-to-pfn') tfc.addMapping("direct", pfn, lfn, mapping_type = 'pfn-to-lfn') f = tempfile.NamedTemporaryFile("w+", delete = True) # read / write tfcStr = str(tfc.getXML()) f.write(tfcStr) f.flush() f.seek(0) tfcInstance = readTFC(f.name) out_pfn = tfc.matchLFN("direct", lfn) out_lfn = tfc.matchPFN("direct", pfn) self.assertEqual(lfn, out_lfn, "Error: incorrect matching") self.assertEqual(pfn, out_pfn, "Error: incorrect matching") f.close()
def testRoundTripWithChain(self): """ Test PFN, LFN conversion when rules are chained. """ tfc = TrivialFileCatalog() match = "/+(.*)" result = "/castor/cern.ch/cms/$1" tfc.addMapping("direct", match, result, mapping_type="lfn-to-pfn") match = "(.*)" result = "$1" tfc.addMapping("stageout", match, result, chain="direct", mapping_type="lfn-to-pfn") in_lfn = "/this/is/a/test/lfn" in_pfn = "/castor/cern.ch/cms" + in_lfn out_pfn = tfc.matchLFN("stageout", in_lfn) self.assertNotEqual(out_pfn, in_lfn) self.assertEqual(out_pfn, in_pfn) tfc = TrivialFileCatalog() match = "/+castor/cern\.ch/cms/(.*)" result = "/$1" tfc.addMapping("direct", match, result, mapping_type="pfn-to-lfn") match = "(.*)" result = "$1" tfc.addMapping("stageout", match, result, chain="direct", mapping_type="pfn-to-lfn") out_lfn = tfc.matchPFN("stageout", in_pfn) self.assertEqual(out_lfn, in_lfn)
def testRoundTripWithChain(self): """ Test PFN, LFN conversion when rules are chained. """ tfc = TrivialFileCatalog() match = "/+(.*)" result = "/castor/cern.ch/cms/$1" tfc.addMapping("direct", match, result, mapping_type = "lfn-to-pfn") match = "(.*)" result = "$1" tfc.addMapping("stageout", match, result, chain = "direct", mapping_type = "lfn-to-pfn") in_lfn = "/this/is/a/test/lfn" in_pfn = "/castor/cern.ch/cms" + in_lfn out_pfn = tfc.matchLFN("stageout", in_lfn) self.assertNotEqual(out_pfn, in_lfn) self.assertEqual(out_pfn, in_pfn) tfc = TrivialFileCatalog() match = "/+castor/cern\.ch/cms/(.*)" result = "/$1" tfc.addMapping("direct", match, result, mapping_type = "pfn-to-lfn") match = "(.*)" result = "$1" tfc.addMapping("stageout", match, result, chain = "direct", mapping_type = "pfn-to-lfn") out_lfn = tfc.matchPFN("stageout", in_pfn) self.assertEqual(out_lfn, in_lfn)
def testRoundTripWithChain(self): """ Test PFN, LFN conversion when rules are chained. """ tfc = TrivialFileCatalog() match = "/+(.*)" result = "/castor/cern.ch/cms/$1" tfc.addMapping("direct", match, result, mapping_type = "lfn-to-pfn") match = "(.*)" result = "$1" tfc.addMapping("stageout", match, result, chain = "direct", mapping_type = "lfn-to-pfn") in_lfn = "/T0/hufnagel/store/t0temp/WMAgentTier0Commissioning/A/RECO/v1dh/0000/4CF196F9-E302-E011-8517-0030487CD716.root" in_pfn = "/castor/cern.ch/cms" + in_lfn out_pfn = tfc.matchLFN("stageout", in_lfn) self.assertNotEqual(out_pfn, in_lfn) self.assertEqual(out_pfn, in_pfn) tfc = TrivialFileCatalog() match = "/+castor/cern\.ch/cms/(.*)" result = "/$1" tfc.addMapping("direct", match, result, mapping_type = "pfn-to-lfn") match = "(.*)" result = "$1" tfc.addMapping("stageout", match, result, chain = "direct", mapping_type = "pfn-to-lfn") out_lfn = tfc.matchPFN("stageout", in_pfn) self.assertEqual(out_lfn, in_lfn)