def batchScriptLocal( remoteDir, index ): '''prepare a local version of the batch script, to run using nohup''' script = """#!/bin/bash echo 'running' %s run_cfg.py if [ $? != 0 ]; then echo wrong exit code! removing all root files rm *.root exit 1 fi echo 'sending the job directory back' """ % prog if remoteDir != '': remoteDir = castortools.eosToLFN(remoteDir) script += """ for file in *.root; do newFileName=`echo $file | sed -r -e 's/\./_%s\./'` cmsStageWithFailover.py -f $file $fullFileName eos chmod 755 /eos/cms/$fullFileName done """ % (index, remoteDir) script += 'rm *.root\n' return script
def batchScriptLocal(remoteDir, index): '''prepare a local version of the batch script, to run using nohup''' script = """#!/bin/bash echo 'running' %s run_cfg.py if [ $? != 0 ]; then echo wrong exit code! removing all root files rm *.root exit 1 fi echo 'sending the job directory back' """ % prog if remoteDir != '': remoteDir = castortools.eosToLFN(remoteDir) script += """ for file in *.root; do newFileName=`echo $file | sed -r -e 's/\./_%s\./'` cmsStageWithFailover.py -f $file $fullFileName eos chmod 755 /eos/cms/$fullFileName done """ % (index, remoteDir) script += 'rm *.root\n' return script
def addRootFilesToValidationRunWithAnalyzerOnDataset( self, validationRunID, eosPath, dir_name, analyzer_name, root_file): try: lfnPath = eostools.eosToLFN(eosPath) datasetInfo = self.selectCur.execute( "SELECT dataset_id,file_owner, path_name from " + self.schema_name + ".dataset_details where lfn='%s'" % lfnPath).fetchone() except: print "ERROR - Could not find parent dataset of the analysis in CMGDB" exit(-1) try: self.insertCur.execute( "INSERT INTO " + self.schema_name + ".validation_root_files_details( validation_run_id, dataset_id, dir_name, analyzer_name, root_file_name ) values(%d, %d, '%s', '%s', '%s')" % (validationRunID, datasetInfo[0], dir_name, analyzer_name, root_file)) self.insertConn.commit() return datasetInfo except cx_Oracle.IntegrityError: print 'ERROR - Root file information not added to database' print sys.exc_info()[1] exit(-1)
def createLoggerTemporaryFile( dataset_lfn_dir ): """Build a temporary logger file object and tarfile object to be used when retrieving tags and jobs""" try: logger_file = tempfile.NamedTemporaryFile() os.system("cmsStage -f "+ os.path.join( eostools.eosToLFN(dataset_lfn_dir), "Logger.tgz") + " " + logger_file.name) logger_tar_object = tarfile.open(fileobj = logger_file) if len( logger_tar_object.getmembers() )==0: print "\nERROR: Failed to stage logger file" exit(-1) return logger_tar_object except: print "\nERROR: Failed to stage logger file" exit(-1)
def createLoggerTemporaryFile(dataset_lfn_dir): """Build a temporary logger file object and tarfile object to be used when retrieving tags and jobs""" try: logger_file = tempfile.NamedTemporaryFile() os.system( "cmsStage -f " + os.path.join(eostools.eosToLFN(dataset_lfn_dir), "Logger.tgz") + " " + logger_file.name) logger_tar_object = tarfile.open(fileobj=logger_file) if len(logger_tar_object.getmembers()) == 0: print "\nERROR: Failed to stage logger file" exit(-1) return logger_tar_object except: print "\nERROR: Failed to stage logger file" exit(-1)
def addRootFilesToValidationRunWithAnalyzerOnDataset( self, validationRunID, eosPath, dir_name, analyzer_name, root_file ): try: lfnPath = eostools.eosToLFN(eosPath) datasetInfo = self.selectCur.execute( "SELECT dataset_id,file_owner, path_name from " +self.schema_name+ ".dataset_details where lfn='%s'" % lfnPath ).fetchone() except: print "ERROR - Could not find parent dataset of the analysis in CMGDB" exit( -1 ) try: self.insertCur.execute( "INSERT INTO " +self.schema_name+ ".validation_root_files_details( validation_run_id, dataset_id, dir_name, analyzer_name, root_file_name ) values(%d, %d, '%s', '%s', '%s')" % (validationRunID, datasetInfo[0], dir_name, analyzer_name, root_file ) ) self.insertConn.commit() return datasetInfo except cx_Oracle.IntegrityError: print 'ERROR - Root file information not added to database' print sys.exc_info()[1] exit(-1)
def createLoggerTemporaryFile(self): """Build a temporary logger file object and tarfile object to be used when retrieving tags and jobs""" try: self._logger_file = tempfile.NamedTemporaryFile() lfn = eostools.eosToLFN(self.dataset_details['LFN']) os.system("cmsStage -f "+lfn+"/Logger.tgz "+self._logger_file.name) self._logger_tar_object = tarfile.open(fileobj=self._logger_file) if len(self._logger_tar_object.getmembers())==0: print "\nERROR: Failed to stage logger file" self._logger_tar_object = None self._logger_file = None except: print "\nERROR: Failed to stage logger file" self._logger_tar_object = None self._logger_file = None
def batchScriptCERN(remoteDir, index): '''prepare the LSF version of the batch script, to run on LSF''' script = """#!/bin/bash # sets the queue #BSUB -q 8nm echo 'environment:' echo env ulimit -v 3000000 echo 'copying job dir to worker' cd $CMSSW_BASE/src eval `scramv1 ru -sh` cd - cp -rf $LS_SUBCWD . ls cd `find . -type d | grep /` echo 'running' %s run_cfg.py if [ $? != 0 ]; then echo wrong exit code! removing all root files rm *.root exit 1 fi echo 'sending the job directory back' """ % prog if remoteDir != '': remoteDir = castortools.eosToLFN( remoteDir) #remoteDir.replace('/eos/cms','') script += """ for file in *.root; do newFileName=`echo $file | sed -r -e 's/\./_%s\./'` fullFileName=%s/$newFileName #this does cmsStage, but with retries cmsStageWithFailover.py -f $file $fullFileName #write the files as user readable but not writable eos chmod 755 /eos/cms/$fullFileName done """ % (index, remoteDir) script += 'rm *.root\n' script += 'cp -rf * $LS_SUBCWD\n' return script
def batchScriptCERN( remoteDir, index ): '''prepare the LSF version of the batch script, to run on LSF''' script = """#!/bin/bash # sets the queue #BSUB -q 8nm echo 'environment:' echo env ulimit -v 3000000 echo 'copying job dir to worker' cd $CMSSW_BASE/src eval `scramv1 ru -sh` cd - cp -rf $LS_SUBCWD . ls cd `find . -type d | grep /` echo 'running' %s run_cfg.py if [ $? != 0 ]; then echo wrong exit code! removing all root files rm *.root exit 1 fi echo 'sending the job directory back' """ % prog if remoteDir != '': remoteDir = castortools.eosToLFN(remoteDir) #remoteDir.replace('/eos/cms','') script += """ for file in *.root; do newFileName=`echo $file | sed -r -e 's/\./_%s\./'` fullFileName=%s/$newFileName #this does cmsStage, but with retries cmsStageWithFailover.py -f $file $fullFileName #write the files as user readable but not writable eos chmod 755 /eos/cms/$fullFileName done """ % (index, remoteDir) script += 'rm *.root\n' script += 'cp -rf * $LS_SUBCWD\n' return script
def createLoggerTemporaryFile(self): """Build a temporary logger file object and tarfile object to be used when retrieving tags and jobs""" try: self._logger_file = tempfile.NamedTemporaryFile() lfn = eostools.eosToLFN(self.dataset_details['LFN']) os.system("cmsStage -f " + lfn + "/Logger.tgz " + self._logger_file.name) self._logger_tar_object = tarfile.open(fileobj=self._logger_file) if len(self._logger_tar_object.getmembers()) == 0: print "\nERROR: Failed to stage logger file" self._logger_tar_object = None self._logger_file = None except: print "\nERROR: Failed to stage logger file" self._logger_tar_object = None self._logger_file = None
def stageOut(self, castorDir): castorDir = castortools.eosToLFN( castorDir ) if not castortools.isLFN( castorDir ): print 'cannot stage out, you need to provide an LFN as a destination directory, beginning with /store .' return False if self.dirLocal != None: tgzDir = self.dirLocal + '.tgz' tgzCmd = 'tar -zcvf ' + tgzDir + ' ' + self.dirLocal print tgzCmd os.system( tgzCmd) cmsStage = 'cmsStage -f %s %s' % (tgzDir, castorDir ) print cmsStage os.system( cmsStage ) os.system('rm ' + tgzDir ) self.tgzDirOnCastor = castorDir + '/' + tgzDir else: print 'cannot stage out, the log is not staged in'
def stageOut(self, castorDir): castorDir = castortools.eosToLFN(castorDir) if not castortools.isLFN(castorDir): print "cannot stage out, you need to provide an LFN as a destination directory, beginning with /store ." return False if self.dirLocal != None: tgzDir = self.dirLocal + ".tgz" tgzCmd = "tar -zcvf " + tgzDir + " " + self.dirLocal print tgzCmd os.system(tgzCmd) cmsStage = "cmsStage -f %s %s" % (tgzDir, castorDir) print cmsStage os.system(cmsStage) os.system("rm " + tgzDir) self.tgzDirOnCastor = castorDir + "/" + tgzDir else: print "cannot stage out, the log is not staged in"
lines = file.readlines() for line in lines: line = re.sub("\s+", " ", line) fileown = line.split("%")[0].lstrip().rstrip() dataset = line.split("%")[1].lstrip().rstrip() files = [] if re.search('group', fileown): castor = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown)) + dataset castor2 = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown.strip("_group"))) + dataset else: castor = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown)) + dataset castor2 = eostools.lfnToEOS( castorBaseDir.castorBaseDir(fileown + "_group")) + dataset LFN = eostools.eosToLFN(castor) LFN2 = eostools.eosToLFN(castor2) if eostools.isDirectory(castor): files = burrow(LFN) elif eostools.isDirectory(castor2): files = burrow(LFN2) print dataset for i in files: print "\t" + i #print "Dataset: "+dataset+" not found"
# For multiple file input file = open(args[0], 'r') lines = file.readlines() for line in lines: line = re.sub("\s+", " ", line) fileown = line.split("%")[0].lstrip().rstrip() dataset = line.split("%")[1].lstrip().rstrip() files = [] if re.search('group',fileown): castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown.strip("_group")))+dataset else: castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown+"_group"))+dataset LFN = eostools.eosToLFN(castor) LFN2 = eostools.eosToLFN(castor2) if eostools.isDirectory(castor): files = burrow(LFN) elif eostools.isDirectory(castor2): files = burrow(LFN2) print dataset for i in files: print "\t"+i #print "Dataset: "+dataset+" not found"
#! /bin/env python import sys import CMGTools.Production.eostools as eostools print "remove", sys.argv[1] files = eostools.ls(sys.argv[1]) print "listing", files print "filter", sys.argv[2] files = [f for f in files if sys.argv[2] in f] print "removing", files ok = len(sys.argv) > 3 print "ok", ok if ok: eostools.remove(files) lfn = eostools.eosToLFN(sys.argv[1]) eostools.runEOSCommand(lfn, 'rmdir', '')
#! /bin/env python import sys import CMGTools.Production.eostools as eostools print "remove",sys.argv[1] files=eostools.ls(sys.argv[1]) print "listing",files print "filter",sys.argv[2] files=[f for f in files if sys.argv[2] in f] print "removing",files ok=len(sys.argv)>3 print "ok",ok if ok: eostools.remove( files ) lfn = eostools.eosToLFN(sys.argv[1]) eostools.runEOSCommand(lfn,'rmdir','')