def allSampleInfo(sampleName, listLevel):
if listLevel == 3:
contents = castortools.ls(castorDir)
for c in contents:
print c
#os.system("rfdir %s | awk '{print \"%s/\"$9}'" % (castorDir,castorDir) )
return
print sampleName
if listLevel > 0:
print '------------------------------------------------------------------------------------------------------------'
print 'PFN:'
print castorDir
print 'LFN:'
print castortools.castorToLFN(castorDir)
if listLevel > 1:
contents = castortools.ls(castorDir)
for c in contents:
print c
if listLevel > 0 and localDir != None:
print 'local:'
print localDir
if os.path.isdir(localDir):
if listLevel > 1:
os.system('ls -l ' + localDir)
# print localDir + '*.root'
else:
if listLevel > 0:
print 'TO BE IMPORTED'
if listLevel > 0:
print
print
def allSampleInfo( sampleName, listLevel ):
if listLevel == 3:
contents = castortools.ls(castorDir)
for c in contents:
print c
#os.system("rfdir %s | awk '{print \"%s/\"$9}'" % (castorDir,castorDir) )
return
print sampleName
if listLevel>0:
print '------------------------------------------------------------------------------------------------------------'
print 'PFN:'
print castorDir
print 'LFN:'
print castortools.castorToLFN(castorDir)
if listLevel>1:
contents = castortools.ls(castorDir)
for c in contents:
print c
if listLevel>0 and localDir!=None:
print 'local:'
print localDir
if os.path.isdir( localDir ):
if listLevel>1:
os.system('ls -l ' + localDir )
# print localDir + '*.root'
else:
if listLevel>0:
print 'TO BE IMPORTED'
if listLevel>0:
print
print
usage="""%prog [options] localdir [chunks]
# This script can be used to retrieve the flat trees stored on EOS during production, and linked in a .url file
"""
parser = OptionParser(usage=usage)
parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
parser.add_option("-c", "--continue", dest="continueCopy", action="store_true", default=False, help='Continue downloading if a chunk failed and print a summary at the end')
parser.add_option("-j", dest="njobs", type=int, default=0, help='Number of parallel downloading tasks')
(options, args) = parser.parse_args()
locdir = args[0]
chunks = eostools.ls(locdir)
print chunks
if len(args)>1:
chunks = [c for c in chunks if any([c.endswith('/'+d) for d in args[1:]])]
print 'Will operate on the following chunks:',chunks
tocopy = []
failedDict = {}
for d in chunks:
f = '%s/%s/%s'%(d,options.treeproducername,options.treename)
furl = '%s.url'%f
if os.path.exists(f):
print 'Chunk %s already contains tree root file %s, skipping'%(d,f)
(options, args) = parser.parse_args()
if len(args)<2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = args[1]
if not eostools.isEOS(remdir): raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
# check removed to allow for top-up of tree productions
# if eostools.fileExists('%s/%s' % (remdir,locdir)):
# raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = eostools.ls(locdir)
dsets = [d for d in alldsets if [ fname for fname in eostools.ls(d) if options.friendtreestring in fname]==[] ]
if options.dset: dsets = [d for d in dsets if options.dset in d]
friends = [d for d in alldsets if d not in dsets]
if options.dset: friends = [d for d in friends if options.dset in d]
tocopy = []
for d in dsets:
if eostools.isFile(d): raise RuntimeError, 'File found in local directory.'
if '%s/%s'%(d,options.treeproducername) not in eostools.ls(d): raise RuntimeError, 'Tree producer sub-directory not found.'
fname = d+'/'+options.treeproducername+'/'+options.treename
if (fname not in eostools.ls('%s/%s'%(d,options.treeproducername))) or (not eostools.isFile(fname)): raise RuntimeError, 'Tree file not found.'
tocopy.append( (fname,'%s/%s_%s_%s'%(remdir,d,options.treeproducername,options.treename)) )
for d in friends:
allfriends = eostools.ls(d)
for f in allfriends:
default="tree.root",
help='Name of the tree file')
(options, args) = parser.parse_args()
if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = args[1]
if not eostools.isEOS(remdir):
raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
if eostools.fileExists('%s/%s' % (remdir, locdir)):
raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = eostools.ls(locdir)
dsets = [
d for d in alldsets if [
fname for fname in eostools.ls(d)
if options.friendtreestring in fname
] == []
]
friends = [d for d in alldsets if d not in dsets]
tocopy = []
for d in dsets:
if eostools.isFile(d):
raise RuntimeError, 'File found in local directory.'
if '%s/%s' % (d, options.treeproducername) not in eostools.ls(d):
raise RuntimeError, 'Tree producer sub-directory not found.'
fname = d + '/' + options.treeproducername + '/' + options.treename
help='only print the commands, do not run the copy')
(options, args) = parser.parse_args()
if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'
eosdir = args[0]
remdir = args[1]
listfiles = []
if len(args) == 3: listfiles = open(args[2]).readlines()
if not eostools.isEOS(eosdir):
raise RuntimeError, 'Remote directory should be on EOS.'
if len(listfiles) > 0: files = listfiles
else: files = eostools.ls(eosdir)
if options.select: files = [f for f in files if options.select in f]
if options.remove: files = [f for f in files if options.remove not in f]
tocopy = []
for f in files:
basename = os.path.basename(f)
tocopy.append(basename)
print "Will copy file ", basename
print '\nCopy the above files from %s -> %s. Do you agree? [y/N]\n' % (
eosdir, remdir)
if raw_input() != 'y':
print 'Aborting'
exit()
#! /bin/env python
import sys
import CMGTools.Production.eostools as eostools
print "remove", sys.argv[1]
files = eostools.ls(sys.argv[1])
print "listing", files
print "filter", sys.argv[2]
files = [f for f in files if sys.argv[2] in f]
print "removing", files
ok = len(sys.argv) > 3
print "ok", ok
if ok:
eostools.remove(files)
lfn = eostools.eosToLFN(sys.argv[1])
eostools.runEOSCommand(lfn, 'rmdir', '')
#! /bin/env python
import sys
import CMGTools.Production.eostools as eostools
print "remove",sys.argv[1]
files=eostools.ls(sys.argv[1])
print "listing",files
print "filter",sys.argv[2]
files=[f for f in files if sys.argv[2] in f]
print "removing",files
ok=len(sys.argv)>3
print "ok",ok
if ok:
eostools.remove( files )
lfn = eostools.eosToLFN(sys.argv[1])
eostools.runEOSCommand(lfn,'rmdir','')
if __name__ == "__main__":
usage="""%prog [options] localdir [chunks]
# This script can be used to retrieve the flat trees stored on EOS during production, and linked in a .url file
"""
parser = OptionParser(usage=usage)
parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
(options, args) = parser.parse_args()
locdir = args[0]
chunks = eostools.ls(locdir)
print chunks
if len(args)>1:
chunks = [c for c in chunks if any([c.endswith('/'+d) for d in args[1:]])]
print 'Will operate on the following chunks:',chunks
tocopy = []
for d in chunks:
f = '%s/%s/%s'%(d,options.treeproducername,options.treename)
furl = '%s.url'%f
if os.path.exists(f):
print 'Chunk %s already contains tree root file %s, skipping'%(d,f)
continue
(options, args) = parser.parse_args()
if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = args[1]
if not eostools.isEOS(remdir):
raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
# check removed to allow for top-up of tree productions
# if eostools.fileExists('%s/%s' % (remdir,locdir)):
# raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = eostools.ls(locdir)
dsets = [
d.split("/")[-1] for d in alldsets if options.friendtreedir not in d
]
if options.dset: dsets = [d for d in dsets if options.dset in d]
friends = [
d.split("/")[-1] for d in alldsets if options.friendtreedir in d
]
if options.dset: friends = [d for d in friends if options.dset in d]
tocopy = []
for d in dsets:
if eostools.isFile(d):
raise RuntimeError, 'File found in local directory.'
if '%s/%s/%s' % (locdir, d,
options.treeproducername) not in eostools.ls(locdir +
