def lfnToPFN(path, tfcProt='rfio'):
"""Converts an LFN to a PFN. For example:
/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
->
root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms
This function only checks path, and does not access the storage system.
If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
Otherwise, is uses the default one.
??? what is tfcprot? """
if path.startswith("/store/"):
path = path.replace("/store/", "root://eoscms.cern.ch//eos/cms/store/")
if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
path = path.replace("/pnfs/psi.ch/cms/trivcat/",
"root://t3se01.psi.ch//")
#print "path to cmsFile():", path
entity = cmsIO.cmsFile(path, tfcProt)
# tokens = cmsIO.splitPFN(entity.pfn)
pfn = '%s://%s//%s/' % (entity.protocol, entity.host, entity.path)
pfn = entity.pfn
if tfcProt == 'rfio' and \
entity.path.startswith("/eos/cms/") and \
str(entity.stat()).startswith("Error 3011: Unable to stat"):
pfn.replace("/eos/cms", "/castor/cern.ch/cms")
pfn.replace("eoscms", "castorcms")
return pfn
def lfnToPFN( path, tfcProt = 'rfio'):
"""Converts an LFN to a PFN. For example:
/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21
->
root://eoscms//eos/cms/store/cmst3/user/cbern/CMG/TauPlusX/Run2011A-03Oct2011-v1/AOD/V2/PAT_CMG_V2_4_0/H2TAUTAU_Nov21?svcClass=cmst3&stageHost=castorcms
This function only checks path, and does not access the storage system.
If the path is in /store/cmst3, it assumes that the CMST3 svcClass is to be used.
Otherwise, is uses the default one.
??? what is tfcprot? """
if path.startswith("/store/"):
path = path.replace("/store/","root://eoscms.cern.ch//eos/cms/store/")
if path.startswith("/pnfs/psi.ch/cms/trivcat/"):
path = path.replace("/pnfs/psi.ch/cms/trivcat/","root://t3se01.psi.ch//")
#print "path to cmsFile():", path
entity = cmsIO.cmsFile( path, tfcProt )
# tokens = cmsIO.splitPFN(entity.pfn)
pfn = '%s://%s//%s/' % (entity.protocol,entity.host,entity.path)
pfn = entity.pfn
if tfcProt == 'rfio' and \
entity.path.startswith("/eos/cms/") and \
str(entity.stat()).startswith("Error 3011: Unable to stat"):
pfn.replace("/eos/cms","/castor/cern.ch/cms")
pfn.replace("eoscms","castorcms")
return pfn
def openTFile(url):
from ROOT import TFile
## File on eos
if url.startswith('/store/'):
cmsf = cmsFile(url, 'rfio')
if not cmsf.isfile(): ## check existence
return None
url = cmsf.pfn
elif not os.path.exists(url): ## check existence
return None
rootFile = TFile.Open(url)
try:
if rootFile.IsZombie(): return None
except ReferenceError:
## Failed to open url (file doesn't exist)
return None
return rootFile
def openTFile(url):
from ROOT import TFile
## File on eos
if url.startswith('/store/'):
cmsf = cmsFile(url, 'rfio')
if not cmsf.isfile(): ## check existence
return None
url = cmsf.pfn
elif not os.path.exists(url): ## check existence
return None
rootFile = TFile.Open(url)
try:
if rootFile.IsZombie(): return None
except ReferenceError:
## Failed to open url (file doesn't exist)
return None
return rootFile
def isEOSFile(path, tfcProt="rfio"):
"""Returns True if path is a file or directory stored on EOS (checks for path existence)
??? This function does not behave well if passed a non EOS path...
returns lots of error messages like:
>>> eostools.isEOSFile('/store/asdfasfd')
Command (['ls', '/', 's', 't', 'o', 'r', 'e', '/', 'a', 's', 'd', 'f', 'a', 's', 'f', 'd', '/store']) failed with return code: 2
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory
ls: s: No such file or directory
ls: t: No such file or directory
ls: o: No such file or directory
ls: r: No such file or directory
ls: e: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: d: No such file or directory
ls: f: No such file or directory
ls: a: No such file or directory
ls: s: No such file or directory
ls: f: No such file or directory
ls: d: No such file or directory
ls: /store: No such file or directory
False
"""
lfn = eosToLFN(path)
entity = cmsIO.cmsFile(os.path.dirname(lfn), tfcProt)
return lfn in entity.ls(lfn)
def getCMSPfn(path, protocol='rfio'):
cmsf = cmsFile(url, protocol)
return cmsf.pfn
def checkMissingFiles(inDir, jsonUrl):
"""
Loop over json inputs and check existence of files.
Also checks if files have a reasonable size (> 1kB)
"""
jsonFile = open(jsonUrl,'r')
procList = json.load(jsonFile,encoding = 'utf-8').items()
# Make a survey of *all* existing plots
total_expected = 0
missing_files = []
suspicious_files = []
recovered_files = []
protocol = 'local'
if inDir.startswith('/store/'):
protocol = 'rfio'
cmsInDir = cmsFile(inDir, protocol)
if not cmsInDir.isdir():
print inDir, "is not a directory"
return False
for proc in procList:
for desc in proc[1]:
data = desc['data']
isData = desc.get('isdata',False)
mctruthmode = desc.get('mctruthmode')
for d in data:
dtag = d.get('dtag','')
split = d.get('split',1)
for segment in range(0,split):
eventsFile = dtag
if split > 1:
eventsFile = dtag + '_' + str(segment)
if mctruthmode:
eventsFile += '_filt%d' % mctruthmode
filename = eventsFile+'.root'
rootFileUrl = inDir+'/'+filename
total_expected += 1
cmsInRootFile = cmsFile(rootFileUrl, protocol)
if not cmsInRootFile.isfile():
missing_files.append(filename)
elif (cmsInRootFile.size() < 1024):
suspicious_files.append(filename)
else:
tfile = openTFile(rootFileUrl)
try:
if tfile.TestBit(ROOT.TFile.kRecovered):
recovered_files.append(filename)
tfile.Close()
except AttributeError:
suspicious_files.append(filename)
continue
print 20*'-'
if len(missing_files):
print "Missing the following files:"
print "(%d out of %d expected)"% (len(missing_files), total_expected)
for filename in missing_files:
print filename
else:
print "NO MISSING FILES!"
print 20*'-'
if len(suspicious_files):
print "The following files are suspicious (< 1kB size or zombie):"
print "(%d out of %d expected)"% (len(suspicious_files), total_expected)
for filename in suspicious_files:
print filename
print 20*'-'
if len(recovered_files):
print "The following files are recovered:"
print "(%d out of %d expected)"% (len(recovered_files), total_expected)
for filename in recovered_files:
print filename
print 20*'-'
def checkMissingFiles(inDir, jsonUrl):
"""
Loop over json inputs and check existence of files.
Also checks if files have a reasonable size (> 1kB)
"""
jsonFile = open(jsonUrl, 'r')
procList = json.load(jsonFile, encoding='utf-8').items()
# Make a survey of *all* existing plots
total_expected = 0
missing_files = []
suspicious_files = []
protocol = 'local'
if inDir.startswith('/store/'):
protocol = 'rfio'
cmsInDir = cmsFile(inDir, protocol)
if not cmsInDir.isdir():
print inDir, "is not a directory"
return False
for proc in procList:
for desc in proc[1]:
data = desc['data']
isData = getByLabel(desc, 'isdata', False)
mctruthmode = getByLabel(desc, 'mctruthmode')
for d in data:
dtag = getByLabel(d, 'dtag', '')
split = getByLabel(d, 'split', 1)
for segment in range(0, split):
eventsFile = dtag
if split > 1:
eventsFile = dtag + '_' + str(segment)
if mctruthmode:
eventsFile += '_filt%d' % mctruthmode
filename = eventsFile + '.root'
rootFileUrl = inDir + '/' + filename
total_expected += 1
cmsInRootFile = cmsFile(rootFileUrl, protocol)
if not cmsInRootFile.isfile():
missing_files.append(filename)
elif (cmsInRootFile.size() < 1024):
suspicious_files.append(filename)
continue
print 20 * '-'
if len(missing_files):
print "Missing the following files:"
print "(%d out of %d expected)" % (len(missing_files), total_expected)
for filename in missing_files:
print filename
else:
print "NO MISSING FILES!"
print 20 * '-'
if len(suspicious_files):
print "The following files are suspicious (< 1kB size):"
print "(%d out of %d expected)" % (len(suspicious_files),
total_expected)
for filename in suspicious_files:
print filename
print 20 * '-'
def getCMSPfn(path, protocol='rfio'):
cmsf = cmsFile(url, protocol)
return cmsf.pfn
