def submit(jobgroup, job, period):
global options
global inbox
# read files from directory
# call program and record both stderr and stdout
filelist = []
section = jobgroup+':'+period
try:
for thisdir in ara.config.get(section, job).split(','):
print 'Adding files from directory', thisdir
# run over files on dCache?
if thisdir.startswith("dcap:/"):
# list files on dcache
path=thisdir[5:]
files = ara.uberftpls(path)
for (size, filename) in files:
filelist.append(ara.config.get('Grid', 'dcap') + path + '/' + filename)
else:
filespec = os.path.abspath(os.path.expanduser(thisdir)) + '/*.root'
files = ara.getCommandOutput2('ls ' + filespec)
for filename in files.splitlines():
filelist.append(filename)
except ConfigParser.NoSectionError:
print "Could not find section %s in main configuration file %s" % ( section , options.cfgfile )
print "You must create this section and add entries to it"
return False
# create directory
basedir = ara.config.get('Analysis', 'basedir').rstrip('/')
myDir = basedir + '/' + options.selection + '/' + period
mkdir_p(myDir)
# need to go into this directory - submit job from there, output will get here
os.chdir(myDir)
# determine file splitting
print 'Found', len(filelist), 'files'
list_of_lists = ara.split_in_jobs(filelist, options.nsplit)
n = 0
for files in list_of_lists:
print "Job #", n, ": ", len(files), " files"
executable = './analyzer'
jobName = job + '_' + str(n)
outputFile = jobName + '.root'
outbox = [ outputFile ]
arguments = job + " " + outputFile + " analyzer.cfg " + " ".join(files)
submit_condor_with_wrapper(executable, arguments,
",".join(inbox), ",".join(outbox),
jobName)
time.sleep(1)
# Next file
n += 1
return True
def main():
defaultOutputFileName = "CutFlow.C"
usage = "usage: %prog [options] input-files"
optParser = optparse.OptionParser(usage)
optParser.add_option("-c", "--config", dest="cfgfile",
help="global configuration file",
default=ara.defaultConfigFileName)
optParser.add_option("-f", "--file", dest="outputfile",
help="output file name",
default=defaultOutputFileName)
(options, args) = optParser.parse_args()
if len(args) < 1:
optParser.print_help()
return 1
ara.config.read(options.cfgfile)
# grep input files for cut flow
files = " ".join(args)
command = "grep 'Fill(\"cutflow\"' " + files + " | cut -d'\"' -f4"
stages = ara.getCommandOutput2(command)
# insert C++ quotes
stages = '"' + '","'.join(stages.splitlines()) + '"'
# open output file
outputFile = open(options.outputfile, "w")
outputFile.write(cutFlowTemplate % stages)
outputFile.close()
return 0
def main():
usage = "usage: %prog [options] number_of_files"
optParser = optparse.OptionParser(usage)
defaultdir="."
optParser.add_option("-c", "--config", dest="cfgfile",
help="global configuration file",
default=ara.defaultConfigFileName)
optParser.add_option("-d", "--dir", dest="dir",
help="directory with ROOT files to merge",
default=defaultdir)
(options, args) = optParser.parse_args()
ara.config.read(options.cfgfile)
# user must give path as argument
if len(args) != 1:
print "You must give the number of files to merge as argument, use -h for help"
return 1
try:
nmerge = int(args[0])
except ValueError:
print "You must give the number of files to merge as argument, use -h for help"
return 1
if (nmerge <= 1):
print "Argument must be a positive number > 1"
return 2
# get all ROOT files in directory
filelist = os.listdir(os.path.abspath(options.dir))
localfiles = [ ]
for filename in filelist:
if filename.endswith(".root"):
localfiles.append(filename);
nfiles = len(localfiles)
print "Found", nfiles , "ROOT files."
# Determine ROOT compression level
compressionlevel = 9
if (len(localfiles) > 0):
info = ara.getCommandOutput2("file " + localfiles[0])
try:
compressionlevel = ((info.split(":"))[2])[1]
except:
pass
print "Using compression level", compressionlevel
counter = 0
while counter*nmerge < nfiles:
start = counter*nmerge
end = min((counter+1)*nmerge, nfiles)
merge(localfiles[start:end], counter, compressionlevel)
counter = counter + 1
return 0
def resubmit(filename, period):
global options
basedir = ara.config.get('Analysis', 'basedir')
myDir = basedir + '/' + options.selection + '/' + period
os.chdir(myDir)
condor_jobfile = myDir+'/'+filename+"_condor.cfg"
ara.wait_for_jobs(options.njobs)
print "Resubmitting", condor_jobfile
rc = ara.getCommandOutput2("condor_submit " + condor_jobfile)
return rc
def join(job, period):
global options
basedir = ara.config.get('Analysis', 'basedir')
myDir = basedir + '/' + options.selection + '/' + period
os.chdir(myDir)
# input file
try:
filelist = ara.getCommandOutput2('ls ' + myDir + '/' + job + '_*.root').splitlines()
except RuntimeError:
print "ERROR: No result files found"
return False
# get file dates
lastDate = 0
for inputFile in filelist:
file_stats = os.stat(inputFile)
lastDate = max(file_stats[stat.ST_MTIME], file_stats[stat.ST_CTIME], lastDate)
# output file
joinFile = myDir + '/' + job + ".root"
joinDate = 0
try:
file_stats = os.stat(joinFile)
joinDate = max(file_stats[stat.ST_MTIME], file_stats[stat.ST_CTIME], joinDate)
except OSError:
pass
# Do not join if joinFile already newer than input files
if (joinDate > lastDate and not options.force):
return True
# Change environment for ROOT if necessary (takes some time)
if options.rootversion_workaround:
command = "source /afs/cern.ch/sw/lcg/external/gcc/4.3.2/x86_64-slc5/setup.sh ; source /afs/cern.ch/sw/lcg/app/releases/ROOT/5.32.01/x86_64-slc5-gcc43-opt/root/bin/thisroot.sh ; hadd -f "
else:
command = "hadd -f "
# now join
if isinstance (filelist, str):
os.system(command + job + ".root " + " " + filelist)
else:
os.system(command + job + ".root " + " ".join(filelist))
return True
def submit_condor_job(executable, arguments, inbox, outbox, jobname):
global options
scriptdir = os.environ['ARASYS'] + '/config'
condor_template_file = open(scriptdir + '/' + options.template)
condor_template = condor_template_file.read()
stderr = "_".join((jobname, "stderr.log"))
stdout = "_".join((jobname, "stdout.log"))
cfgFile = "_".join((jobname, "condor.cfg"))
repMap = {}
repMap["executable"] = executable
repMap["arguments"] = arguments
repMap["inputfiles"] = inbox
repMap["stdout"] = stdout
repMap["stderr"] = stderr
repMap["log"] = "condor.log"
repMap["outputfiles"] = outbox
content = condor_template % repMap
jobfile = open(cfgFile, "w")
jobfile.write(content)
jobfile.close()
ara.wait_for_jobs(ara.get_maximum_jobs(options.njobs))
rc = ara.getCommandOutput2("condor_submit " + cfgFile)
return rc
def main():
global options
usage = """usage: %prog [options] selection [jobgroup] [pattern]
Will check output of CONDOR jobs and join files in given jobgroup matching the
specified pattern. "jobgroup" defaults to the default job group named
"default", and "pattern" defaults to "all", i.e. all jobs in the given
jobgroup will be checked and joined."""
optParser = optparse.OptionParser(usage)
defaultnjobs=20
optParser.add_option("-c", "--config", dest="cfgfile",
help="global configuration file",
default=ara.defaultConfigFileName)
optParser.add_option("-f", "--force-overwrite", dest="force",
help="Force overwriting of existing .root files",
action="store_true",
default=False)
optParser.add_option("-r", "--resubmit", dest="resubmit", action="store_true",
help="Resubmit failed jobs")
optParser.add_option("-n", "--njobs", dest="njobs",
help="how many jobs to run at once if resubmitting",
default=defaultnjobs)
(options, args) = optParser.parse_args()
if len(args) < 1 or len(args) > 3:
optParser.print_help()
return 1
try:
ROOTSYS=os.environ['ROOTSYS']
except KeyError:
print("You must setup correct ROOTSYS for AdvancedROOTAnalyzer to work")
return 1
# get number of jobs
try:
options.njobs = int(options.njobs)
except ValueError:
print "Number of jobs has to be a number"
return 1
if options.njobs <= 0:
print "Number of jobs must be greater than zero"
return 2
# read global configuration file
ara.config.read(options.cfgfile)
# setup local arguments
options.selection = args[0]
options.jobgroup = 'default'
options.pattern = 'all'
if len(args) >= 2:
options.jobgroup = args[1]
if len(args) >= 3:
options.pattern = args[2]
# Check ROOT version
options.rootversion_workaround = False
rootversion = ara.getCommandOutput2("root-config --version").strip()
first = rootversion.split("/")[0]
fix = rootversion.split("/")[1]
major = int( first.split(".")[0])
minor = int( first.split(".")[1])
if major < 5 or (major == 5 and minor < 32):
print "You are using an old root version", rootversion
print "Enabling workaround for calling hadd"
options.rootversion_workaround = True
errors = False
try:
# check if this is a job group
section = 'jobgroup:'+options.jobgroup
items = ara.config.items(section)
# OK, is a job group
print "Checking and joining jobs from job group", options.jobgroup
for (period, jobs) in items:
print "Job group period", period, "consists of jobs", jobs
for job in jobs.split(','):
if options.pattern in job or options.pattern == 'all':
print "Collecting job", job
errors = check_join(options.jobgroup, job, period) and errors
except ConfigParser.NoSectionError:
print section, "is not a job group."
print "You must configure job groups in file %s" % ( options.cfgfile )
return 1
if errors:
print "*******************************************************************************"
print "* There were errors, check script output *"
print "*******************************************************************************"
return 1
return 0
