Ejemplo n.º 1
0
        "--force",
        dest="force",
        action="store_true",
        default=False,
        help=
        "Skip files or directories that don't contain trees, without raising an error"
    )
    (options, args) = parser.parse_args()
    if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'

    locdir = args[0]
    remdir = os.path.join(args[1], os.path.basename(locdir))

    if not eostools.isEOS(remdir):
        raise RuntimeError, 'Remote directory should be on EOS.'
    if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
        raise RuntimeError, 'The local directory that should contain the trees does not exist.'

# check removed to allow for top-up of tree productions
#	if eostools.fileExists('%s/%s' % (remdir,locdir)):
#	    raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'

    alldsets = [p for p in glob.glob(locdir + "/*") if os.path.isdir(p)]
    if not options.allowSymlinks:
        symlinks = [d for d in alldsets if os.path.islink(d)]
        if symlinks:
            print "The following directories are symlinks and will not be considered (run with --allowSymlinks to include them): ", ", ".join(
                map(os.path.basename, symlinks))
            alldsets = [d for d in alldsets if not os.path.islink(d)]
    dsets = [
        d for d in alldsets if [
Ejemplo n.º 2
0
        'String identifying friend trees (must be contained in the root file name)'
    )
    parser.add_option("-T",
                      dest="treename",
                      type='string',
                      default="tree.root",
                      help='Name of the tree file')
    (options, args) = parser.parse_args()
    if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'

    locdir = args[0]
    remdir = args[1]

    if not eostools.isEOS(remdir):
        raise RuntimeError, 'Remote directory should be on EOS.'
    if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
        raise RuntimeError, 'The local directory that should contain the trees does not exist.'
    if eostools.fileExists('%s/%s' % (remdir, locdir)):
        raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'

    alldsets = eostools.ls(locdir)
    dsets = [
        d for d in alldsets if [
            fname for fname in eostools.ls(d)
            if options.friendtreestring in fname
        ] == []
    ]
    friends = [d for d in alldsets if d not in dsets]

    tocopy = []
    for d in dsets:
Ejemplo n.º 3
0
"""

	parser = OptionParser(usage=usage)
	parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
	parser.add_option("-f", dest="friendtreestring", type='string', default="evVarFriend", help='String identifying friend trees (must be contained in the root file name)')
	parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
	parser.add_option("--dset", dest="dset", type='string', default=None, help='Name of the dataset to process')
	(options, args) = parser.parse_args()
	if len(args)<2: raise RuntimeError, 'Expecting at least two arguments'
	
	locdir = args[0]
	remdir = args[1]
	
	
	if not eostools.isEOS(remdir): raise RuntimeError, 'Remote directory should be on EOS.'
	if (not eostools.fileExists(locdir)) or eostools.isFile(locdir): 
	    raise RuntimeError, 'The local directory that should contain the trees does not exist.'

# check removed to allow for top-up of tree productions
#	if eostools.fileExists('%s/%s' % (remdir,locdir)):
#	    raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'
	
	alldsets = eostools.ls(locdir)
	dsets = [d for d in alldsets if [ fname for fname in eostools.ls(d) if options.friendtreestring in fname]==[] ]
	if options.dset: dsets = [d for d in dsets if options.dset in d]
	friends = [d for d in alldsets if d not in dsets]
	if options.dset: friends = [d for d in friends if options.dset in d]
	
	tocopy = []
	for d in dsets:
	    if eostools.isFile(d): raise RuntimeError, 'File found in local directory.'
Ejemplo n.º 4
0
    dest = args[1]
    if eostools.isDirectory(dest):
        dest = os.path.join(dest, os.path.basename(args[0]))

    sleep_lengths = [1, 10, 60, 600, 1800]
    return_code = 0
    for i in xrange(5):

        #sleep for a while before running
        time.sleep(sleep_lengths[i])

        try:
            #run cmsStage
            print 'cmsStage %s [%d/5]' % (' '.join(argv), i + 1)
            main(argv)

        except SystemExit, e:
            print "cmsStage exited with code '%s'. Retrying... [%d/5]" % (
                str(e), i + 1)
            return_code = e.code

        #sleep again before checking
        time.sleep(3)

        if eostools.fileExists(dest) and eostools.isFile(dest):
            if source.size() == destination.size():
                return_code = 0
                break

    sys.exit(return_code)
Ejemplo n.º 5
0
    locdir = args[0]
    chunks = eostools.ls(locdir)

    print chunks

    if len(args) > 1:
        chunks = [
            c for c in chunks if any([c.endswith('/' + d) for d in args[1:]])
        ]

    print 'Will operate on the following chunks:', chunks

    tocopy = []
    for d in chunks:
        f = '%s/%s/%s' % (d, options.treeproducername, options.treename)
        furl = '%s.url' % f
        if os.path.exists(f):
            print 'Chunk %s already contains tree root file %s, skipping' % (d,
                                                                             f)
            continue
        if not os.path.exists(furl):
            raise RuntimeError, 'Chunk %s does not contain url file %s' % (
                d, furl)
        with open(furl, 'r') as _furl:
            rem = _furl.readline().replace('root://eoscms.cern.ch/',
                                           '').replace('\n', '')
            if not eostools.isFile(rem):
                raise RuntimeError, 'Remote file %s not found' % rem
            eostools.xrdcp(rem, f)
	parser = OptionParser(usage=usage)
	parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
	parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
	(options, args) = parser.parse_args()
	
	locdir = args[0]
	chunks = eostools.ls(locdir)

	print chunks

	if len(args)>1:
		chunks = [c for c in chunks if any([c.endswith('/'+d) for d in args[1:]])]

	print 'Will operate on the following chunks:',chunks

	tocopy = []
	for d in chunks:
		f = '%s/%s/%s'%(d,options.treeproducername,options.treename)
		furl = '%s.url'%f
		if os.path.exists(f):
			print 'Chunk %s already contains tree root file %s, skipping'%(d,f)
			continue
		if not os.path.exists(furl):
			raise RuntimeError,'Chunk %s does not contain url file %s'%(d,furl)
		with open(furl,'r') as _furl:
			rem = _furl.readline().replace('root://eoscms.cern.ch/','').replace('\n','')
			if not eostools.isFile(rem):
				raise RuntimeError,'Remote file %s not found'%rem
			eostools.xrdcp(rem,f)

Ejemplo n.º 7
0
    #find the destination LFN
    dest = args[1]
    if eostools.isDirectory(dest):
        dest = os.path.join(dest,os.path.basename(args[0]))
    
    sleep_lengths = [1,10,60,600,1800]
    return_code = 0
    for i in xrange(5):

        #sleep for a while before running
        time.sleep(sleep_lengths[i])

        try:
            #run cmsStage
            print 'cmsStage %s [%d/5]' % (' '.join(argv) , i+1)
            main(argv)

        except SystemExit, e:
            print "cmsStage exited with code '%s'. Retrying... [%d/5]" % ( str(e), i+1 )
            return_code = e.code
        
        #sleep again before checking
        time.sleep(3)

        if eostools.fileExists(dest) and eostools.isFile(dest):
            if source.size() == destination.size():
                return_code = 0
                break

    sys.exit(return_code)
Ejemplo n.º 8
0
                      default="tree.root",
                      help='Name of the tree file')
    parser.add_option("--dset",
                      dest="dset",
                      type='string',
                      default=None,
                      help='Name of the dataset to process')
    (options, args) = parser.parse_args()
    if len(args) < 2: raise RuntimeError, 'Expecting at least two arguments'

    locdir = args[0]
    remdir = args[1]

    if not eostools.isEOS(remdir):
        raise RuntimeError, 'Remote directory should be on EOS.'
    if (not eostools.fileExists(locdir)) or eostools.isFile(locdir):
        raise RuntimeError, 'The local directory that should contain the trees does not exist.'

# check removed to allow for top-up of tree productions
#	if eostools.fileExists('%s/%s' % (remdir,locdir)):
#	    raise RuntimeError, 'The remote EOS directory where the trees should be archived already exists.'

    alldsets = eostools.ls(locdir)
    dsets = [
        d.split("/")[-1] for d in alldsets if options.friendtreedir not in d
    ]
    if options.dset: dsets = [d for d in dsets if options.dset in d]
    friends = [
        d.split("/")[-1] for d in alldsets if options.friendtreedir in d
    ]
    if options.dset: friends = [d for d in friends if options.dset in d]