def lfnToPFN(path, tfcProt='rfio'): """Converts an LFN to a PFN. For example: /store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21 -> root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms This function only checks path, and does not access the storage system. If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used. Otherwise, is uses the default one. ??? what is tfcprot? """ if path.startswith("/store/"): path = path.replace("/store/", "root://eoscms.cern.ch//eos/cms/store/") if path.startswith("/pnfs/psi.ch/cms/trivcat/"): path = path.replace("/pnfs/psi.ch/cms/trivcat/", "root://t3se01.psi.ch//") #print "path to cmsFile():", path entity = cmsIO.cmsFile(path, tfcProt) # tokens = cmsIO.splitPFN(entity.pfn) pfn = '%s://%s//%s/' % (entity.protocol, entity.host, entity.path) pfn = entity.pfn if tfcProt == 'rfio' and \ entity.path.startswith("/eos/cms/") and \ str(entity.stat()).startswith("Error 3011: Unable to stat"): pfn.replace("/eos/cms", "/castor/cern.ch/cms") pfn.replace("eoscms", "castorcms") return pfn
def lfnToPFN( path, tfcProt = 'rfio'): """Converts an LFN to a PFN. For example: /store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21 -> root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms This function only checks path, and does not access the storage system. If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used. Otherwise, is uses the default one. ??? what is tfcprot? """ if path.startswith("/store/"): path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/") if path.startswith("/pnfs/psi.ch/cms/trivcat/"): path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//") #print "path to cmsFile():", path entity = cmsIO.cmsFile( path, tfcProt ) # tokens = cmsIO.splitPFN(entity.pfn) pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path) pfn = entity.pfn if tfcProt == 'rfio' and \ entity.path.startswith("/eos/cms/") and \ str(entity.stat()).startswith("Error 3011: Unable to stat"): pfn.replace("/eos/cms","/castor/cern.ch/cms") pfn.replace("eoscms","castorcms") return pfn
def openTFile(url): from ROOT import TFile ## File on eos if url.startswith('/store/'): cmsf = cmsFile(url, 'rfio') if not cmsf.isfile(): ## check existence return None url = cmsf.pfn elif not os.path.exists(url): ## check existence return None rootFile = TFile.Open(url) try: if rootFile.IsZombie(): return None except ReferenceError: ## Failed to open url (file doesn't exist) return None return rootFile
def isEOSFile(path, tfcProt="rfio"): """Returns True if path is a file or directory stored on EOS (checks for path existence) ??? This function does not behave well if passed a non EOS path... returns lots of error messages like: >>> eostools.isEOSFile('/store/asdfasfd') Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2 ls: s: No such file or directory ls: t: No such file or directory ls: o: No such file or directory ls: r: No such file or directory ls: e: No such file or directory ls: a: No such file or directory ls: s: No such file or directory ls: d: No such file or directory ls: f: No such file or directory ls: a: No such file or directory ls: s: No such file or directory ls: f: No such file or directory ls: d: No such file or directory ls: /store: No such file or directory ls: s: No such file or directory ls: t: No such file or directory ls: o: No such file or directory ls: r: No such file or directory ls: e: No such file or directory ls: a: No such file or directory ls: s: No such file or directory ls: d: No such file or directory ls: f: No such file or directory ls: a: No such file or directory ls: s: No such file or directory ls: f: No such file or directory ls: d: No such file or directory ls: /store: No such file or directory False """ lfn = eosToLFN(path) entity = cmsIO.cmsFile(os.path.dirname(lfn), tfcProt) return lfn in entity.ls(lfn)
def getCMSPfn(path, protocol='rfio'): cmsf = cmsFile(url, protocol) return cmsf.pfn
def checkMissingFiles(inDir, jsonUrl): """ Loop over json inputs and check existence of files. Also checks if files have a reasonable size (> 1kB) """ jsonFile = open(jsonUrl,'r') procList = json.load(jsonFile,encoding = 'utf-8').items() # Make a survey of *all* existing plots total_expected = 0 missing_files = [] suspicious_files = [] recovered_files = [] protocol = 'local' if inDir.startswith('/store/'): protocol = 'rfio' cmsInDir = cmsFile(inDir, protocol) if not cmsInDir.isdir(): print inDir, "is not a directory" return False for proc in procList: for desc in proc[1]: data = desc['data'] isData = desc.get('isdata',False) mctruthmode = desc.get('mctruthmode') for d in data: dtag = d.get('dtag','') split = d.get('split',1) for segment in range(0,split): eventsFile = dtag if split > 1: eventsFile = dtag + '_' + str(segment) if mctruthmode: eventsFile += '_filt%d' % mctruthmode filename = eventsFile+'.root' rootFileUrl = inDir+'/'+filename total_expected += 1 cmsInRootFile = cmsFile(rootFileUrl, protocol) if not cmsInRootFile.isfile(): missing_files.append(filename) elif (cmsInRootFile.size() < 1024): suspicious_files.append(filename) else: tfile = openTFile(rootFileUrl) try: if tfile.TestBit(ROOT.TFile.kRecovered): recovered_files.append(filename) tfile.Close() except AttributeError: suspicious_files.append(filename) continue print 20*'-' if len(missing_files): print "Missing the following files:" print "(%d out of %d expected)"% (len(missing_files), total_expected) for filename in missing_files: print filename else: print "NO MISSING FILES!" print 20*'-' if len(suspicious_files): print "The following files are suspicious (< 1kB size or zombie):" print "(%d out of %d expected)"% (len(suspicious_files), total_expected) for filename in suspicious_files: print filename print 20*'-' if len(recovered_files): print "The following files are recovered:" print "(%d out of %d expected)"% (len(recovered_files), total_expected) for filename in recovered_files: print filename print 20*'-'
def checkMissingFiles(inDir, jsonUrl): """ Loop over json inputs and check existence of files. Also checks if files have a reasonable size (> 1kB) """ jsonFile = open(jsonUrl, 'r') procList = json.load(jsonFile, encoding='utf-8').items() # Make a survey of *all* existing plots total_expected = 0 missing_files = [] suspicious_files = [] protocol = 'local' if inDir.startswith('/store/'): protocol = 'rfio' cmsInDir = cmsFile(inDir, protocol) if not cmsInDir.isdir(): print inDir, "is not a directory" return False for proc in procList: for desc in proc[1]: data = desc['data'] isData = getByLabel(desc, 'isdata', False) mctruthmode = getByLabel(desc, 'mctruthmode') for d in data: dtag = getByLabel(d, 'dtag', '') split = getByLabel(d, 'split', 1) for segment in range(0, split): eventsFile = dtag if split > 1: eventsFile = dtag + '_' + str(segment) if mctruthmode: eventsFile += '_filt%d' % mctruthmode filename = eventsFile + '.root' rootFileUrl = inDir + '/' + filename total_expected += 1 cmsInRootFile = cmsFile(rootFileUrl, protocol) if not cmsInRootFile.isfile(): missing_files.append(filename) elif (cmsInRootFile.size() < 1024): suspicious_files.append(filename) continue print 20 * '-' if len(missing_files): print "Missing the following files:" print "(%d out of %d expected)" % (len(missing_files), total_expected) for filename in missing_files: print filename else: print "NO MISSING FILES!" print 20 * '-' if len(suspicious_files): print "The following files are suspicious (< 1kB size):" print "(%d out of %d expected)" % (len(suspicious_files), total_expected) for filename in suspicious_files: print filename print 20 * '-'