Пример #1
0
 def __init__(self, src, dest):
     
     #this seems to matter to xrd
     if not dest.endswith(os.sep):
         dest = dest+ os.sep
     
     self.src = src
     self.dest = dest
     
     #self.src_files = [f for f in castortools.listFiles(src, rec = False) if f and castortools.isFile(f)]
     #self.dest_files = [f for f in castortools.listFiles(dest, rec = False) if f and castortools.isFile(f)]
     self.src_files = [(f[4],f[1]) for f in castortools.listFiles(src, rec = False, full_info = True) if f]
     self.dest_files = [(f[4],f[1]) for f in castortools.listFiles(dest, rec = False, full_info = True) if f]
     
     self.problem_files = []
     
     #clean the files if they are already present
     src_dist = {}
     for s in self.src_files:
         src_dist[os.path.basename(s[0])] = s
     
     for d in self.dest_files:
         base = os.path.basename(d[0])
         if src_dist.has_key(base):
             s = src_dist[base]
             if d[1] != s[1]:
                 self.problem(s,'File exists in destination, but with different file size.')
             print "Removing '%s' from copy list" % s[0]
             self.src_files.remove(s)
Пример #2
0
def getListOfFiles(expr, baseDir, filePattern):
    if baseDir.startswith("/store"):
        return [
            "root://eoscms/%s" % f for f in eostools.listFiles(
                expr.format(baseDir=baseDir, filePattern=""))
            if fnmatch.fnmatch(f, filePattern)
        ]
    else:
        return expr.format(baseDir=baseDir, filePattern=filePattern)
Пример #3
0
def getListOfFiles(baseDir, filePattern, expr="{baseDir}/{filePattern}"):
    if baseDir.startswith("/store"):
        return [
            "root://eoscms/%s" % f for f in eostools.listFiles(
                expr.format(baseDir=baseDir, filePattern=""))
            if os.path.basename(f) == filePattern
            or fnmatch.fnmatch(f, filePattern)
        ]
    else:
        return glob.glob(expr.format(baseDir=baseDir, filePattern=filePattern))
Пример #4
0
 def getFilesFromEOS(self,name,dataset,path,pattern=".*root"):
     from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
     if "%" in path: path = path % dataset;
     try:
         files = getDatasetFromCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern))
     except IOError:
         files = [ 'root://eoscms.cern.ch/'+x for x in eostools.listFiles('/eos/cms'+path) if re.match(pattern,x) ]
         if len(files) == 0:
             raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (name,path,pattern)
         writeDatasetToCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern), files)
     return files
Пример #5
0
 def getFilesFromEOS(self,name,dataset,path,pattern=".*root"):
     from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
     if "%" in path: path = path % dataset;
     try:
         files = getDatasetFromCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern))
     except IOError:
         files = [ 'root://eoscms.cern.ch/'+x for x in eostools.listFiles('/eos/cms'+path) if re.match(pattern,x) ] 
         if len(files) == 0:
             raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (name,path,pattern)
         writeDatasetToCache('EOS%{path}%{pattern}.pck'.format(path = path.replace('/','_'), pattern = pattern), files)
     return files
Пример #6
0
    def getDataFilesFromLocal(self, name, path, pattern=".*root"):
        from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
        print path
        files = [
            x for x in eostools.listFiles(path, True) if re.match(pattern, x)
        ]
        if len(files) == 0:
            raise RuntimeError, "ERROR making component %s: no files found under %s tching '%s'" % (
                name, path, pattern)

        return files
Пример #7
0
 def getFilesFromPSI(self, name, dataset, path, pattern=".*root"):
     from CMGTools.Production.dataset import getDatasetFromCache, writeDatasetToCache
     if "%" in path: path = path % dataset
     try:
         files = getDatasetFromCache('PSI%{path}%{pattern}.pck'.format(
             path=path.replace('/', '_'), pattern=pattern))
     except IOError:
         files = [
             'root://t3se01.psi.ch//' +
             x.replace("/pnfs/psi.ch/cms/trivcat/", "")
             for x in eostools.listFiles('/pnfs/psi.ch/cms/trivcat/' + path)
             if re.match(pattern, x)
         ]
         if len(files) == 0:
             raise RuntimeError, "ERROR making component %s: no files found under %s matching '%s'" % (
                 name, path, pattern)
         writeDatasetToCache(
             'PSI%{path}%{pattern}.pck'.format(path=path.replace('/', '_'),
                                               pattern=pattern), files)
     return files
Пример #8
0
 def listFiles(self,dir):
     """Recursively list a file or directory on castor"""
     return castortools.listFiles(dir,self.options.resursive)
Пример #9
0
def getListOfFiles( baseDir, filePattern, expr="{baseDir}/{filePattern}"):
    if baseDir.startswith("/store"):
        return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format( baseDir=baseDir, filePattern="" ) )
                 if os.path.basename(f) == filePattern or fnmatch.fnmatch(f,filePattern) ]
    else:
        return glob.glob(expr.format( baseDir=baseDir, filePattern=filePattern))
Пример #10
0
def getListOfFiles(expr, baseDir, filePattern):
    if baseDir.startswith("/store"):
        return [ "root://eoscms/%s" % f for f in eostools.listFiles( expr.format( baseDir=baseDir, filePattern="" ) ) if fnmatch.fnmatch(f,filePattern) ]
    else:
        return expr.format( baseDir=baseDir, filePattern=filePattern ) 
Пример #11
0
 def listFiles(self, dir):
     """Recursively list a file or directory on castor"""
     return castortools.listFiles(dir, self.options.resursive)