def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name, 'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name), fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname, path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN( new_name) os.remove(nname) else: pathhash = path.replace('/', '.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % ( castor_path, hashed_name)
def publish(self, report): """Publish a file""" for path in report['PathList']: _, name = tempfile.mkstemp('.txt', text=True) json.dump(report, file(name,'w'), sort_keys=True, indent=4) fname = '%s_%s.txt' % (self.parent, report['DateCreated']) #rename the file locally - TODO: This is a potential problem nname = os.path.join(os.path.dirname(name),fname) os.rename(name, nname) castor_path = castortools.lfnToCastor(path) new_name = '%s/%s' % (castor_path, fname) castortools.xrdcp(nname,path) time.sleep(1) if castortools.fileExists(new_name): #castortools.move(old_name, new_name) #castortools.chmod(new_name, '644') print "File published: '%s'" % castortools.castorToLFN(new_name) os.remove(nname) else: pathhash = path.replace('/','.') hashed_name = 'PublishToFileSystem-%s-%s' % (pathhash, fname) shutil.move(nname, hashed_name) print >> sys.stderr, "Cannot write to directory '%s' - written to local file '%s' instead." % (castor_path, hashed_name)
def _runIt(args): rem,f = args print 'Downloading %s...'%rem eostools.xrdcp(rem,f)
print 'Chunk %s already contains tree root file %s, skipping'%(d,f) continue if not os.path.exists(furl): if (options.continueCopy): print 'Chunk %s does not contain url file %s' % (d, furl) failedDict[d] = furl continue else: raise RuntimeError,'Chunk %s does not contain url file %s'%(d,furl) with open(furl,'r') as _furl: rem = _furl.readline().replace('root://eoscms.cern.ch/','').replace('\n','') if not os.path.isfile(rem): raise RuntimeError,'Remote file %s not found'%rem if options.njobs>0: tocopy.append((rem,f)) else: eostools.xrdcp(rem,f) if options.njobs>0: def _runIt(args): rem,f = args print 'Downloading %s...'%rem eostools.xrdcp(rem,f) Pool(options.njobs).map(_runIt,tocopy) if (options.continueCopy and (len(failedDict.keys()) > 0)): print "="*100 print "Summary of failed download attempts (%d in total):" % len(failedDict.keys()) for d, furl in failedDict.iteritems(): print 'Chunk %s does not contain url file %s' % (d, furl)
raise RuntimeError, '.url file already exists for %s (use --force to force a new archival)' % task[ 0] print 'Will create EOS directory %s and copy the following files:\n' % remdir for task in tocopy: print '%s -> %s' % task print '\nDo you agree? [y/N]\n' if raw_input() != 'y': print 'Aborting' exit() eostools.mkdir(remdir) if not eostools.fileExists(remdir): raise RuntimeError, 'Impossible to create remote directory.' for task in tocopy: eostools.xrdcp(task[0], task[1]) fcmd = open(task[0] + ".url", "w") fcmd.write("root://eoscms.cern.ch/%s\n" % task[1]) fcmd.close() print 'Copied %.2f GB to EOS\n' % eostools.eosDirSize(remdir) print 'Verifying checksums:\n' problem = False for task in tocopy: lcheck = eostools.fileChecksum(task[0]) rcheck = eostools.fileChecksum(task[1]) ok = (lcheck == rcheck) print task[0], lcheck, rcheck, ('OK' if ok else 'ERROR') if not ok: problem = True if problem: raise RuntimeError, 'CHECKSUM ERROR DETECTED !!!'
for task in tocopy: if eostools.fileExists(task[0]+".url"): raise RuntimeError, '.url file already exists.' newdir='%s/%s'%(remdir,locdir) print 'Will create EOS directory %s and copy the following files:\n'%newdir for task in tocopy: print '%s -> %s' % task print '\nDo you agree? [y/N]\n' if raw_input()!='y': print 'Aborting' exit() eostools.mkdir(newdir) if not eostools.fileExists(newdir): raise RuntimeError, 'Impossible to create remote directory.' for task in tocopy: eostools.xrdcp(task[0],task[1]) fcmd = open(task[0]+".url","w") fcmd.write("root://eoscms.cern.ch/%s\n" % task[1]) fcmd.close() print 'Copied %.2f GB to EOS\n' % eostools.eosDirSize(newdir) print 'Verifying checksums:\n' problem = False for task in tocopy: lcheck = eostools.fileChecksum(task[0]) rcheck = eostools.fileChecksum(task[1]) ok = (lcheck==rcheck) print task[0],lcheck,rcheck,('OK' if ok else 'ERROR') if not ok: problem = True if problem: raise RuntimeError, 'CHECKSUM ERROR DETECTED !!!'
#! /bin/env python import sys import CMGTools.Production.eostools as eostools # print sys.argv[1] eostools.xrdcp( sys.argv[1], sys.argv[2] ) # eostools.listFiles('/eos/cms/store/cmst3/user/cbern/Tests/', rec=True) # sys.exit(0)
#! /bin/env python import sys import CMGTools.Production.eostools as eostools # print sys.argv[1] eostools.xrdcp(sys.argv[1], sys.argv[2]) # eostools.listFiles('/eos/cms/store/cmst3/user/cbern/Tests/', rec=True) # sys.exit(0)
parser = OptionParser(usage=usage) parser.add_option("-t", dest="treeproducername", type='string', default="myTreeProducer", help='Name of the tree producer module') parser.add_option("-T", dest="treename", type='string', default="tree.root", help='Name of the tree file') (options, args) = parser.parse_args() locdir = args[0] chunks = eostools.ls(locdir) print chunks if len(args)>1: chunks = [c for c in chunks if any([c.endswith('/'+d) for d in args[1:]])] print 'Will operate on the following chunks:',chunks tocopy = [] for d in chunks: f = '%s/%s/%s'%(d,options.treeproducername,options.treename) furl = '%s.url'%f if os.path.exists(f): print 'Chunk %s already contains tree root file %s, skipping'%(d,f) continue if not os.path.exists(furl): raise RuntimeError,'Chunk %s does not contain url file %s'%(d,furl) with open(furl,'r') as _furl: rem = _furl.readline().replace('root://eoscms.cern.ch/','').replace('\n','') if not eostools.isFile(rem): raise RuntimeError,'Remote file %s not found'%rem eostools.xrdcp(rem,f)