コード例 #1
0
    def publish(self, report):
        """Publish a file"""
        for path in report['PathList']:
            _, name = tempfile.mkstemp('.txt', text=True)
            json.dump(report, file(name, 'w'), sort_keys=True, indent=4)

            fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
            #rename the file locally - TODO: This is a potential problem
            nname = os.path.join(os.path.dirname(name), fname)
            os.rename(name, nname)

            castor_path = castortools.lfnToCastor(path)
            new_name = '%s/%s' % (castor_path, fname)
            castortools.xrdcp(nname, path)
            time.sleep(1)

            if castortools.fileExists(new_name):

                #castortools.move(old_name, new_name)
                #castortools.chmod(new_name, '644')

                print "File published: '%s'" % castortools.castorToLFN(
                    new_name)
                os.remove(nname)
            else:
                pathhash = path.replace('/', '.')
                hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
                shutil.move(nname, hashed_name)
                print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (
                    castor_path, hashed_name)
コード例 #2
0
    def publish(self, report):
        """Publish a file"""
        for path in report['PathList']:
            _, name = tempfile.mkstemp('.txt', text=True)
            json.dump(report, file(name,'w'), sort_keys=True, indent=4)
            
            fname = '%s_%s.txt' % (self.parent, report['DateCreated'])
            #rename the file locally - TODO: This is a potential problem
            nname = os.path.join(os.path.dirname(name),fname)
            os.rename(name, nname)
            
            castor_path = castortools.lfnToCastor(path)
            new_name = '%s/%s' % (castor_path, fname)
            castortools.xrdcp(nname,path)
            time.sleep(1)
            
            if castortools.fileExists(new_name):
                
                #castortools.move(old_name, new_name)
                #castortools.chmod(new_name, '644')

                print "File published: '%s'" % castortools.castorToLFN(new_name)
                os.remove(nname)
            else:
                pathhash = path.replace('/','.')
                hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname)
                shutil.move(nname, hashed_name)
                print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
コード例 #3
0
		def _runIt(args):
			rem,f = args
			print 'Downloading %s...'%rem
			eostools.xrdcp(rem,f)
コード例 #4
0
			print 'Chunk %s already contains tree root file %s, skipping'%(d,f)
			continue
		if not os.path.exists(furl):
			if (options.continueCopy):
				print 'Chunk %s does not contain url file %s' % (d, furl)
				failedDict[d] = furl
				continue
			else:
				raise RuntimeError,'Chunk %s does not contain url file %s'%(d,furl)
		with open(furl,'r') as _furl:
			rem = _furl.readline().replace('root://eoscms.cern.ch/','').replace('\n','')
			if not os.path.isfile(rem):
				raise RuntimeError,'Remote file %s not found'%rem
			if options.njobs>0:
				tocopy.append((rem,f))
			else:
				eostools.xrdcp(rem,f)

	if options.njobs>0:
		def _runIt(args):
			rem,f = args
			print 'Downloading %s...'%rem
			eostools.xrdcp(rem,f)
		Pool(options.njobs).map(_runIt,tocopy)

	if (options.continueCopy and (len(failedDict.keys()) > 0)):
		print "="*100
		print "Summary of failed download attempts (%d in total):" % len(failedDict.keys())
		for d, furl in failedDict.iteritems():
			print 'Chunk %s does not contain url file %s' % (d, furl)
コード例 #5
0
                raise RuntimeError, '.url file already exists for %s (use --force to force a new archival)' % task[
                    0]
    print 'Will create EOS directory %s and copy the following files:\n' % remdir
    for task in tocopy:
        print '%s -> %s' % task

    print '\nDo you agree? [y/N]\n'
    if raw_input() != 'y':
        print 'Aborting'
        exit()

    eostools.mkdir(remdir)
    if not eostools.fileExists(remdir):
        raise RuntimeError, 'Impossible to create remote directory.'
    for task in tocopy:
        eostools.xrdcp(task[0], task[1])
        fcmd = open(task[0] + ".url", "w")
        fcmd.write("root://eoscms.cern.ch/%s\n" % task[1])
        fcmd.close()
    print 'Copied %.2f GB to EOS\n' % eostools.eosDirSize(remdir)

    print 'Verifying checksums:\n'
    problem = False
    for task in tocopy:
        lcheck = eostools.fileChecksum(task[0])
        rcheck = eostools.fileChecksum(task[1])
        ok = (lcheck == rcheck)
        print task[0], lcheck, rcheck, ('OK' if ok else 'ERROR')
        if not ok: problem = True
    if problem:
        raise RuntimeError, 'CHECKSUM ERROR DETECTED !!!'
コード例 #6
0
	for task in tocopy:
	    if eostools.fileExists(task[0]+".url"): raise RuntimeError, '.url file already exists.'
	
	newdir='%s/%s'%(remdir,locdir)
	print 'Will create EOS directory %s and copy the following files:\n'%newdir
	for task in tocopy: print '%s -> %s' % task
	
	print '\nDo you agree? [y/N]\n'
	if raw_input()!='y':
	    print 'Aborting'
	    exit()
	
	eostools.mkdir(newdir)
	if not eostools.fileExists(newdir): raise RuntimeError, 'Impossible to create remote directory.'
	for task in tocopy:
	    eostools.xrdcp(task[0],task[1])
	    fcmd = open(task[0]+".url","w")
	    fcmd.write("root://eoscms.cern.ch/%s\n" % task[1])
	    fcmd.close()
	print 'Copied %.2f GB to EOS\n' % eostools.eosDirSize(newdir)

	print 'Verifying checksums:\n'
	problem = False
	for task in tocopy:
		lcheck = eostools.fileChecksum(task[0])
		rcheck = eostools.fileChecksum(task[1])
		ok = (lcheck==rcheck)
		print task[0],lcheck,rcheck,('OK' if ok else 'ERROR')
		if not ok: problem = True
	if problem:
		raise RuntimeError, 'CHECKSUM ERROR DETECTED !!!'
コード例 #7
0
ファイル: cmgCp.py プロジェクト: anantoni/CMG
#! /bin/env python
import sys

import CMGTools.Production.eostools as eostools

# print sys.argv[1]
eostools.xrdcp( sys.argv[1], sys.argv[2] )

# eostools.listFiles('/eos/cms/store/cmst3/user/cbern/Tests/', rec=True)

# sys.exit(0)
コード例 #8
0
ファイル: cmgCp.py プロジェクト: jeyserma/cmgtools-lite
#! /bin/env python
import sys

import CMGTools.Production.eostools as eostools

# print sys.argv[1]
eostools.xrdcp(sys.argv[1], sys.argv[2])

# eostools.listFiles('/eos/cms/store/cmst3/user/cbern/Tests/', rec=True)

# sys.exit(0)
コード例 #9
0
	parser = OptionParser(usage=usage)
	parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module')
	parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file')
	(options, args) = parser.parse_args()
	
	locdir = args[0]
	chunks = eostools.ls(locdir)

	print chunks

	if len(args)>1:
		chunks = [c for c in chunks if any([c.endswith('/'+d) for d in args[1:]])]

	print 'Will operate on the following chunks:',chunks

	tocopy = []
	for d in chunks:
		f = '%s/%s/%s'%(d,options.treeproducername,options.treename)
		furl = '%s.url'%f
		if os.path.exists(f):
			print 'Chunk %s already contains tree root file %s, skipping'%(d,f)
			continue
		if not os.path.exists(furl):
			raise RuntimeError,'Chunk %s does not contain url file %s'%(d,furl)
		with open(furl,'r') as _furl:
			rem = _furl.readline().replace('root://eoscms.cern.ch/','').replace('\n','')
			if not eostools.isFile(rem):
				raise RuntimeError,'Remote file %s not found'%rem
			eostools.xrdcp(rem,f)