def __init__(self, src, dest): #this seems to matter to xrd if not dest.endswith(os.sep): dest = dest+ os.sep self.src = src self.dest = dest #self.src_files = [f for f in castortools.listFiles(src, rec = False) if f and castortools.isFile(f)] #self.dest_files = [f for f in castortools.listFiles(dest, rec = False) if f and castortools.isFile(f)] self.src_files = [(f[4],f[1]) for f in castortools.listFiles(src, rec = False, full_info = True) if f] self.dest_files = [(f[4],f[1]) for f in castortools.listFiles(dest, rec = False, full_info = True) if f] self.problem_files = [] #clean the files if they are already present src_dist = {} for s in self.src_files: src_dist[os.path.basename(s[0])] = s for d in self.dest_files: base = os.path.basename(d[0]) if src_dist.has_key(base): s = src_dist[base] if d[1] != s[1]: self.problem(s,'File exists in destination, but with different file size.') print "Removing '%s' from copy list" % s[0] self.src_files.remove(s)
def getListOfFiles(expr, baseDir, filePattern): if baseDir.startswith("/store"): return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format(baseDir=baseDir, filePattern="")) if fnmatch.fnmatch(f, filePattern) ] else: return expr.format(baseDir=baseDir, filePattern=filePattern)
def getListOfFiles(baseDir, filePattern, expr="{baseDir}/{filePattern}"): if baseDir.startswith("/store"): return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format(baseDir=baseDir, filePattern="")) if os.path.basename(f) == filePattern or fnmatch.fnmatch(f, filePattern) ] else: return glob.glob(expr.format(baseDir=baseDir, filePattern=filePattern))
def getFilesFromEOS(self,name,dataset,path,pattern=".*root"): from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache if "%" in path: path = path % dataset; try: files = getDatasetFromCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern)) except IOError: files = [ 'root://eoscms.cern.ch/'+x for x in eostools.listFiles('/eos/cms'+path) if re.match(pattern,x) ] if len(files) == 0: raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (name,path,pattern) writeDatasetToCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern), files) return files
def getDataFilesFromLocal(self, name, path, pattern=".*root"): from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache print path files = [ x for x in eostools.listFiles(path, True) if re.match(pattern, x) ] if len(files) == 0: raise RuntimeError, "ERROR making component %s: no files found under %s tching '%s'" % ( name, path, pattern) return files
def getFilesFromPSI(self, name, dataset, path, pattern=".*root"): from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache if "%" in path: path = path % dataset try: files = getDatasetFromCache('PSI%{path}%{pattern}.pck'.format( path=path.replace('/', '_'), pattern=pattern)) except IOError: files = [ 'root://t3se01.psi.ch//' + x.replace("/pnfs/psi.ch/cms/trivcat/", "") for x in eostools.listFiles('/pnfs/psi.ch/cms/trivcat/' + path) if re.match(pattern, x) ] if len(files) == 0: raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % ( name, path, pattern) writeDatasetToCache( 'PSI%{path}%{pattern}.pck'.format(path=path.replace('/', '_'), pattern=pattern), files) return files
def listFiles(self,dir): """Recursively list a file or directory on castor""" return castortools.listFiles(dir,self.options.resursive)
def getListOfFiles( baseDir, filePattern, expr="{baseDir}/{filePattern}"): if baseDir.startswith("/store"): return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format( baseDir=baseDir, filePattern="" ) ) if os.path.basename(f) == filePattern or fnmatch.fnmatch(f,filePattern) ] else: return glob.glob(expr.format( baseDir=baseDir, filePattern=filePattern))
def getListOfFiles(expr, baseDir, filePattern): if baseDir.startswith("/store"): return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format( baseDir=baseDir, filePattern="" ) ) if fnmatch.fnmatch(f,filePattern) ] else: return expr.format( baseDir=baseDir, filePattern=filePattern )
def listFiles(self, dir): """Recursively list a file or directory on castor""" return castortools.listFiles(dir, self.options.resursive)