def __init__(self, src, dest):
#this seems to matter to xrd
if not dest.endswith(os.sep):
dest = dest+ os.sep
self.src = src
self.dest = dest
#self.src_files = [f for f in castortools.listFiles(src, rec = False) if f and castortools.isFile(f)]
#self.dest_files = [f for f in castortools.listFiles(dest, rec = False) if f and castortools.isFile(f)]
self.src_files = [(f[4],f[1]) for f in castortools.listFiles(src, rec = False, full_info = True) if f]
self.dest_files = [(f[4],f[1]) for f in castortools.listFiles(dest, rec = False, full_info = True) if f]
self.problem_files = []
#clean the files if they are already present
src_dist = {}
for s in self.src_files:
src_dist[os.path.basename(s[0])] = s
for d in self.dest_files:
base = os.path.basename(d[0])
if src_dist.has_key(base):
s = src_dist[base]
if d[1] != s[1]:
self.problem(s,'File exists in destination, but with different file size.')
print "Removing '%s' from copy list" % s[0]
self.src_files.remove(s)
def getListOfFiles(expr, baseDir, filePattern):
if baseDir.startswith("/store"):
return [
"root://eoscms/%s" % f for f in eostools.listFiles(
expr.format(baseDir=baseDir, filePattern=""))
if fnmatch.fnmatch(f, filePattern)
]
else:
return expr.format(baseDir=baseDir, filePattern=filePattern)
def getListOfFiles(baseDir, filePattern, expr="{baseDir}/{filePattern}"):
if baseDir.startswith("/store"):
return [
"root://eoscms/%s" % f for f in eostools.listFiles(
expr.format(baseDir=baseDir, filePattern=""))
if os.path.basename(f) == filePattern
or fnmatch.fnmatch(f, filePattern)
]
else:
return glob.glob(expr.format(baseDir=baseDir, filePattern=filePattern))
def getFilesFromEOS(self,name,dataset,path,pattern=".*root"):
from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
if "%" in path: path = path % dataset;
try:
files = getDatasetFromCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern))
except IOError:
files = [ 'root://eoscms.cern.ch/'+x for x in eostools.listFiles('/eos/cms'+path) if re.match(pattern,x) ]
if len(files) == 0:
raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (name,path,pattern)
writeDatasetToCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern), files)
return files
def getFilesFromEOS(self,name,dataset,path,pattern=".*root"):
from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
if "%" in path: path = path % dataset;
try:
files = getDatasetFromCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern))
except IOError:
files = [ 'root://eoscms.cern.ch/'+x for x in eostools.listFiles('/eos/cms'+path) if re.match(pattern,x) ]
if len(files) == 0:
raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (name,path,pattern)
writeDatasetToCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern), files)
return files
def getDataFilesFromLocal(self, name, path, pattern=".*root"):
from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
print path
files = [
x for x in eostools.listFiles(path, True) if re.match(pattern, x)
]
if len(files) == 0:
raise RuntimeError, "ERROR making component %s: no files found under %s tching '%s'" % (
name, path, pattern)
return files
def getFilesFromPSI(self, name, dataset, path, pattern=".*root"):
from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
if "%" in path: path = path % dataset
try:
files = getDatasetFromCache('PSI%{path}%{pattern}.pck'.format(
path=path.replace('/', '_'), pattern=pattern))
except IOError:
files = [
'root://t3se01.psi.ch//' +
x.replace("/pnfs/psi.ch/cms/trivcat/", "")
for x in eostools.listFiles('/pnfs/psi.ch/cms/trivcat/' + path)
if re.match(pattern, x)
]
if len(files) == 0:
raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (
name, path, pattern)
writeDatasetToCache(
'PSI%{path}%{pattern}.pck'.format(path=path.replace('/', '_'),
pattern=pattern), files)
return files
def listFiles(self,dir):
"""Recursively list a file or directory on castor"""
return castortools.listFiles(dir,self.options.resursive)
def getListOfFiles( baseDir, filePattern, expr="{baseDir}/{filePattern}"):
if baseDir.startswith("/store"):
return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format( baseDir=baseDir, filePattern="" ) )
if os.path.basename(f) == filePattern or fnmatch.fnmatch(f,filePattern) ]
else:
return glob.glob(expr.format( baseDir=baseDir, filePattern=filePattern))
def getListOfFiles(expr, baseDir, filePattern):
if baseDir.startswith("/store"):
return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format( baseDir=baseDir, filePattern="" ) ) if fnmatch.fnmatch(f,filePattern) ]
else:
return expr.format( baseDir=baseDir, filePattern=filePattern )
def listFiles(self, dir):
"""Recursively list a file or directory on castor"""
return castortools.listFiles(dir, self.options.resursive)