"--force",
dest="force",
action="store_true",
default=False,
help=
"Skip files or directories that don't contain trees, without raising an error"
)
(options, args) = parser.parse_args()
if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = os.path.join(args[1], os.path.basename(locdir))
if not eostools.isEOS(remdir):
raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
# check removed to allow for top-up of tree productions
# if eostools.fileExists('%s/%s' % (remdir,locdir)):
# raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = [p for p in glob.glob(locdir + "/*") if os.path.isdir(p)]
if not options.allowSymlinks:
symlinks = [d for d in alldsets if os.path.islink(d)]
if symlinks:
print "The following directories are symlinks and will not be considered (run with --allowSymlinks to include them): ", ", ".join(
map(os.path.basename, symlinks))
alldsets = [d for d in alldsets if not os.path.islink(d)]
dsets = [
d for d in alldsets if [
'String identifying friend trees (must be contained in the root file name)'
)
parser.add_option("-T",
dest="treename",
type='string',
default="tree.root",
help='Name of the tree file')
(options, args) = parser.parse_args()
if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = args[1]
if not eostools.isEOS(remdir):
raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
if eostools.fileExists('%s/%s' % (remdir, locdir)):
raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = eostools.ls(locdir)
dsets = [
d for d in alldsets if [
fname for fname in eostools.ls(d)
if options.friendtreestring in fname
] == []
]
friends = [d for d in alldsets if d not in dsets]
tocopy = []
for d in dsets:
"""
parser = OptionParser(usage=usage)
parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
parser.add_option("-f", dest="friendtreestring", type='string', default="evVarFriend", help='String identifying friend trees (must be contained in the root file name)')
parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
parser.add_option("--dset", dest="dset", type='string', default=None, help='Name of the dataset to process')
(options, args) = parser.parse_args()
if len(args)<2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = args[1]
if not eostools.isEOS(remdir): raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
# check removed to allow for top-up of tree productions
# if eostools.fileExists('%s/%s' % (remdir,locdir)):
# raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = eostools.ls(locdir)
dsets = [d for d in alldsets if [ fname for fname in eostools.ls(d) if options.friendtreestring in fname]==[] ]
if options.dset: dsets = [d for d in dsets if options.dset in d]
friends = [d for d in alldsets if d not in dsets]
if options.dset: friends = [d for d in friends if options.dset in d]
tocopy = []
for d in dsets:
if eostools.isFile(d): raise RuntimeError, 'File found in local directory.'
dest = args[1]
if eostools.isDirectory(dest):
dest = os.path.join(dest, os.path.basename(args[0]))
sleep_lengths = [1, 10, 60, 600, 1800]
return_code = 0
for i in xrange(5):
#sleep for a while before running
time.sleep(sleep_lengths[i])
try:
#run cmsStage
print 'cmsStage %s [%d/5]' % (' '.join(argv), i + 1)
main(argv)
except SystemExit, e:
print "cmsStage exited with code '%s'. Retrying... [%d/5]" % (
str(e), i + 1)
return_code = e.code
#sleep again before checking
time.sleep(3)
if eostools.fileExists(dest) and eostools.isFile(dest):
if source.size() == destination.size():
return_code = 0
break
sys.exit(return_code)
locdir = args[0]
chunks = eostools.ls(locdir)
print chunks
if len(args) > 1:
chunks = [
c for c in chunks if any([c.endswith('/' + d) for d in args[1:]])
]
print 'Will operate on the following chunks:', chunks
tocopy = []
for d in chunks:
f = '%s/%s/%s' % (d, options.treeproducername, options.treename)
furl = '%s.url' % f
if os.path.exists(f):
print 'Chunk %s already contains tree root file %s, skipping' % (d,
f)
continue
if not os.path.exists(furl):
raise RuntimeError, 'Chunk %s does not contain url file %s' % (
d, furl)
with open(furl, 'r') as _furl:
rem = _furl.readline().replace('root://eoscms.cern.ch/',
'').replace('\n', '')
if not eostools.isFile(rem):
raise RuntimeError, 'Remote file %s not found' % rem
eostools.xrdcp(rem, f)
parser = OptionParser(usage=usage)
parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
(options, args) = parser.parse_args()
locdir = args[0]
chunks = eostools.ls(locdir)
print chunks
if len(args)>1:
chunks = [c for c in chunks if any([c.endswith('/'+d) for d in args[1:]])]
print 'Will operate on the following chunks:',chunks
tocopy = []
for d in chunks:
f = '%s/%s/%s'%(d,options.treeproducername,options.treename)
furl = '%s.url'%f
if os.path.exists(f):
print 'Chunk %s already contains tree root file %s, skipping'%(d,f)
continue
if not os.path.exists(furl):
raise RuntimeError,'Chunk %s does not contain url file %s'%(d,furl)
with open(furl,'r') as _furl:
rem = _furl.readline().replace('root://eoscms.cern.ch/','').replace('\n','')
if not eostools.isFile(rem):
raise RuntimeError,'Remote file %s not found'%rem
eostools.xrdcp(rem,f)
#find the destination LFN
dest = args[1]
if eostools.isDirectory(dest):
dest = os.path.join(dest,os.path.basename(args[0]))
sleep_lengths = [1,10,60,600,1800]
return_code = 0
for i in xrange(5):
#sleep for a while before running
time.sleep(sleep_lengths[i])
try:
#run cmsStage
print 'cmsStage %s [%d/5]' % (' '.join(argv) , i+1)
main(argv)
except SystemExit, e:
print "cmsStage exited with code '%s'. Retrying... [%d/5]" % ( str(e), i+1 )
return_code = e.code
#sleep again before checking
time.sleep(3)
if eostools.fileExists(dest) and eostools.isFile(dest):
if source.size() == destination.size():
return_code = 0
break
sys.exit(return_code)
default="tree.root",
help='Name of the tree file')
parser.add_option("--dset",
dest="dset",
type='string',
default=None,
help='Name of the dataset to process')
(options, args) = parser.parse_args()
if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'
locdir = args[0]
remdir = args[1]
if not eostools.isEOS(remdir):
raise RuntimeError, 'Remote directory should be on EOS.'
if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
raise RuntimeError, 'The local directory that should contain the trees does not exist.'
# check removed to allow for top-up of tree productions
# if eostools.fileExists('%s/%s' % (remdir,locdir)):
# raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
alldsets = eostools.ls(locdir)
dsets = [
d.split("/")[-1] for d in alldsets if options.friendtreedir not in d
]
if options.dset: dsets = [d for d in dsets if options.dset in d]
friends = [
d.split("/")[-1] for d in alldsets if options.friendtreedir in d
]
if options.dset: friends = [d for d in friends if options.dset in d]
