def batchScriptLocal( remoteDir, index ):
'''prepare a local version of the batch script, to run using nohup'''
script = """#!/bin/bash
echo 'running'
%s run_cfg.py
if [ $? != 0 ]; then
echo wrong exit code! removing all root files
rm *.root
exit 1
fi
echo 'sending the job directory back'
""" % prog
if remoteDir != '':
remoteDir = castortools.eosToLFN(remoteDir)
script += """
for file in *.root; do
newFileName=`echo $file | sed -r -e 's/\./_%s\./'`
cmsStageWithFailover.py -f $file $fullFileName
eos chmod 755 /eos/cms/$fullFileName
done
""" % (index, remoteDir)
script += 'rm *.root\n'
return script
def batchScriptLocal(remoteDir, index):
'''prepare a local version of the batch script, to run using nohup'''
script = """#!/bin/bash
echo 'running'
%s run_cfg.py
if [ $? != 0 ]; then
echo wrong exit code! removing all root files
rm *.root
exit 1
fi
echo 'sending the job directory back'
""" % prog
if remoteDir != '':
remoteDir = castortools.eosToLFN(remoteDir)
script += """
for file in *.root; do
newFileName=`echo $file | sed -r -e 's/\./_%s\./'`
cmsStageWithFailover.py -f $file $fullFileName
eos chmod 755 /eos/cms/$fullFileName
done
""" % (index, remoteDir)
script += 'rm *.root\n'
return script
def addRootFilesToValidationRunWithAnalyzerOnDataset(
self, validationRunID, eosPath, dir_name, analyzer_name,
root_file):
try:
lfnPath = eostools.eosToLFN(eosPath)
datasetInfo = self.selectCur.execute(
"SELECT dataset_id,file_owner, path_name from " +
self.schema_name +
".dataset_details where lfn='%s'" % lfnPath).fetchone()
except:
print "ERROR - Could not find parent dataset of the analysis in CMGDB"
exit(-1)
try:
self.insertCur.execute(
"INSERT INTO " + self.schema_name +
".validation_root_files_details( validation_run_id, dataset_id, dir_name, analyzer_name, root_file_name ) values(%d, %d, '%s', '%s', '%s')"
% (validationRunID, datasetInfo[0], dir_name, analyzer_name,
root_file))
self.insertConn.commit()
return datasetInfo
except cx_Oracle.IntegrityError:
print 'ERROR - Root file information not added to database'
print sys.exc_info()[1]
exit(-1)
def createLoggerTemporaryFile( dataset_lfn_dir ):
"""Build a temporary logger file object and tarfile object to be used
when retrieving tags and jobs"""
try:
logger_file = tempfile.NamedTemporaryFile()
os.system("cmsStage -f "+ os.path.join( eostools.eosToLFN(dataset_lfn_dir), "Logger.tgz") + " " + logger_file.name)
logger_tar_object = tarfile.open(fileobj = logger_file)
if len( logger_tar_object.getmembers() )==0:
print "\nERROR: Failed to stage logger file"
exit(-1)
return logger_tar_object
except:
print "\nERROR: Failed to stage logger file"
exit(-1)
def createLoggerTemporaryFile(dataset_lfn_dir):
"""Build a temporary logger file object and tarfile object to be used
when retrieving tags and jobs"""
try:
logger_file = tempfile.NamedTemporaryFile()
os.system(
"cmsStage -f " +
os.path.join(eostools.eosToLFN(dataset_lfn_dir), "Logger.tgz") +
" " + logger_file.name)
logger_tar_object = tarfile.open(fileobj=logger_file)
if len(logger_tar_object.getmembers()) == 0:
print "\nERROR: Failed to stage logger file"
exit(-1)
return logger_tar_object
except:
print "\nERROR: Failed to stage logger file"
exit(-1)
def addRootFilesToValidationRunWithAnalyzerOnDataset( self, validationRunID, eosPath, dir_name, analyzer_name, root_file ):
try:
lfnPath = eostools.eosToLFN(eosPath)
datasetInfo = self.selectCur.execute( "SELECT dataset_id,file_owner, path_name from " +self.schema_name+ ".dataset_details where lfn='%s'" % lfnPath
).fetchone()
except:
print "ERROR - Could not find parent dataset of the analysis in CMGDB"
exit( -1 )
try:
self.insertCur.execute( "INSERT INTO " +self.schema_name+ ".validation_root_files_details( validation_run_id, dataset_id, dir_name, analyzer_name, root_file_name ) values(%d, %d, '%s', '%s', '%s')" % (validationRunID, datasetInfo[0], dir_name, analyzer_name, root_file ) )
self.insertConn.commit()
return datasetInfo
except cx_Oracle.IntegrityError:
print 'ERROR - Root file information not added to database'
print sys.exc_info()[1]
exit(-1)
def createLoggerTemporaryFile(self):
"""Build a temporary logger file object and tarfile
object to be used when retrieving tags and jobs"""
try:
self._logger_file = tempfile.NamedTemporaryFile()
lfn = eostools.eosToLFN(self.dataset_details['LFN'])
os.system("cmsStage -f "+lfn+"/Logger.tgz "+self._logger_file.name)
self._logger_tar_object = tarfile.open(fileobj=self._logger_file)
if len(self._logger_tar_object.getmembers())==0:
print "\nERROR: Failed to stage logger file"
self._logger_tar_object = None
self._logger_file = None
except:
print "\nERROR: Failed to stage logger file"
self._logger_tar_object = None
self._logger_file = None
def batchScriptCERN(remoteDir, index):
'''prepare the LSF version of the batch script, to run on LSF'''
script = """#!/bin/bash
# sets the queue
#BSUB -q 8nm
echo 'environment:'
echo
env
ulimit -v 3000000
echo 'copying job dir to worker'
cd $CMSSW_BASE/src
eval `scramv1 ru -sh`
cd -
cp -rf $LS_SUBCWD .
ls
cd `find . -type d | grep /`
echo 'running'
%s run_cfg.py
if [ $? != 0 ]; then
echo wrong exit code! removing all root files
rm *.root
exit 1
fi
echo 'sending the job directory back'
""" % prog
if remoteDir != '':
remoteDir = castortools.eosToLFN(
remoteDir) #remoteDir.replace('/eos/cms','')
script += """
for file in *.root; do
newFileName=`echo $file | sed -r -e 's/\./_%s\./'`
fullFileName=%s/$newFileName
#this does cmsStage, but with retries
cmsStageWithFailover.py -f $file $fullFileName
#write the files as user readable but not writable
eos chmod 755 /eos/cms/$fullFileName
done
""" % (index, remoteDir)
script += 'rm *.root\n'
script += 'cp -rf * $LS_SUBCWD\n'
return script
def batchScriptCERN( remoteDir, index ):
'''prepare the LSF version of the batch script, to run on LSF'''
script = """#!/bin/bash
# sets the queue
#BSUB -q 8nm
echo 'environment:'
echo
env
ulimit -v 3000000
echo 'copying job dir to worker'
cd $CMSSW_BASE/src
eval `scramv1 ru -sh`
cd -
cp -rf $LS_SUBCWD .
ls
cd `find . -type d | grep /`
echo 'running'
%s run_cfg.py
if [ $? != 0 ]; then
echo wrong exit code! removing all root files
rm *.root
exit 1
fi
echo 'sending the job directory back'
""" % prog
if remoteDir != '':
remoteDir = castortools.eosToLFN(remoteDir) #remoteDir.replace('/eos/cms','')
script += """
for file in *.root; do
newFileName=`echo $file | sed -r -e 's/\./_%s\./'`
fullFileName=%s/$newFileName
#this does cmsStage, but with retries
cmsStageWithFailover.py -f $file $fullFileName
#write the files as user readable but not writable
eos chmod 755 /eos/cms/$fullFileName
done
""" % (index, remoteDir)
script += 'rm *.root\n'
script += 'cp -rf * $LS_SUBCWD\n'
return script
def createLoggerTemporaryFile(self):
"""Build a temporary logger file object and tarfile
object to be used when retrieving tags and jobs"""
try:
self._logger_file = tempfile.NamedTemporaryFile()
lfn = eostools.eosToLFN(self.dataset_details['LFN'])
os.system("cmsStage -f " + lfn + "/Logger.tgz " +
self._logger_file.name)
self._logger_tar_object = tarfile.open(fileobj=self._logger_file)
if len(self._logger_tar_object.getmembers()) == 0:
print "\nERROR: Failed to stage logger file"
self._logger_tar_object = None
self._logger_file = None
except:
print "\nERROR: Failed to stage logger file"
self._logger_tar_object = None
self._logger_file = None
def stageOut(self, castorDir):
castorDir = castortools.eosToLFN( castorDir )
if not castortools.isLFN( castorDir ):
print 'cannot stage out, you need to provide an LFN as a destination directory, beginning with /store .'
return False
if self.dirLocal != None:
tgzDir = self.dirLocal + '.tgz'
tgzCmd = 'tar -zcvf ' + tgzDir + ' ' + self.dirLocal
print tgzCmd
os.system( tgzCmd)
cmsStage = 'cmsStage -f %s %s' % (tgzDir, castorDir )
print cmsStage
os.system( cmsStage )
os.system('rm ' + tgzDir )
self.tgzDirOnCastor = castorDir + '/' + tgzDir
else:
print 'cannot stage out, the log is not staged in'
def stageOut(self, castorDir):
castorDir = castortools.eosToLFN(castorDir)
if not castortools.isLFN(castorDir):
print "cannot stage out, you need to provide an LFN as a destination directory, beginning with /store ."
return False
if self.dirLocal != None:
tgzDir = self.dirLocal + ".tgz"
tgzCmd = "tar -zcvf " + tgzDir + " " + self.dirLocal
print tgzCmd
os.system(tgzCmd)
cmsStage = "cmsStage -f %s %s" % (tgzDir, castorDir)
print cmsStage
os.system(cmsStage)
os.system("rm " + tgzDir)
self.tgzDirOnCastor = castorDir + "/" + tgzDir
else:
print "cannot stage out, the log is not staged in"
lines = file.readlines()
for line in lines:
line = re.sub("\s+", " ", line)
fileown = line.split("%")[0].lstrip().rstrip()
dataset = line.split("%")[1].lstrip().rstrip()
files = []
if re.search('group', fileown):
castor = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown)) + dataset
castor2 = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown.strip("_group"))) + dataset
else:
castor = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown)) + dataset
castor2 = eostools.lfnToEOS(
castorBaseDir.castorBaseDir(fileown + "_group")) + dataset
LFN = eostools.eosToLFN(castor)
LFN2 = eostools.eosToLFN(castor2)
if eostools.isDirectory(castor):
files = burrow(LFN)
elif eostools.isDirectory(castor2):
files = burrow(LFN2)
print dataset
for i in files:
print "\t" + i
#print "Dataset: "+dataset+" not found"
# For multiple file input
file = open(args[0], 'r')
lines = file.readlines()
for line in lines:
line = re.sub("\s+", " ", line)
fileown = line.split("%")[0].lstrip().rstrip()
dataset = line.split("%")[1].lstrip().rstrip()
files = []
if re.search('group',fileown):
castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset
castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown.strip("_group")))+dataset
else:
castor = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown))+dataset
castor2 = eostools.lfnToEOS(castorBaseDir.castorBaseDir(fileown+"_group"))+dataset
LFN = eostools.eosToLFN(castor)
LFN2 = eostools.eosToLFN(castor2)
if eostools.isDirectory(castor):
files = burrow(LFN)
elif eostools.isDirectory(castor2):
files = burrow(LFN2)
print dataset
for i in files: print "\t"+i
#print "Dataset: "+dataset+" not found"
#! /bin/env python
import sys
import CMGTools.Production.eostools as eostools
print "remove", sys.argv[1]
files = eostools.ls(sys.argv[1])
print "listing", files
print "filter", sys.argv[2]
files = [f for f in files if sys.argv[2] in f]
print "removing", files
ok = len(sys.argv) > 3
print "ok", ok
if ok:
eostools.remove(files)
lfn = eostools.eosToLFN(sys.argv[1])
eostools.runEOSCommand(lfn, 'rmdir', '')
#! /bin/env python
import sys
import CMGTools.Production.eostools as eostools
print "remove",sys.argv[1]
files=eostools.ls(sys.argv[1])
print "listing",files
print "filter",sys.argv[2]
files=[f for f in files if sys.argv[2] in f]
print "removing",files
ok=len(sys.argv)>3
print "ok",ok
if ok:
eostools.remove( files )
lfn = eostools.eosToLFN(sys.argv[1])
eostools.runEOSCommand(lfn,'rmdir','')
