parser = OptionParser('Usage: %prog [options]')
parser.add_option('-l', '--list', type = 'string', dest = 'list', help = 'List of run IDs', default = '')
parser.add_option('-j', '--job', type = 'string', dest = 'job', help = 'Type of job: track, vertex, online, etc.', default = '')
parser.add_option('-c', '--config', type = 'string', dest = 'config', help = 'I/O configuration file', default = '')
parser.add_option('-n', '--njobs', type = 'int', dest = 'nJobs', help = 'Number of jobs to split one run', default = 10)
parser.add_option('-e', '--errlog', type = 'string', dest = 'errlog', help = 'Failed command log', default = 'submitAll_err.log')
parser.add_option('-d', '--debug', action = 'store_true', dest = 'debug', help = 'Enable massive debugging output', default = False)
parser.add_option('-s', '--submit', action = 'store_true', dest = 'submit', help = 'Submit the long jobs in smaller batches', default = False)
parser.add_option('-m', '--monitor', action = 'store_true', dest = 'monitor', help = 'Monitor the long job status', default = False)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
# initialize the configuration
conf = GU.JobConfig(options.config)
# parse the list
jobInfo = [line.strip().split() for line in open(options.list)]
for i in range(len(jobInfo)):
runID = int(re.findall(r'_(\d+)_r', jobInfo[i][1])[0])
runID, outtag, firstEvent, nEvents = GU.getJobAttr(os.path.join(conf.outdir, 'opts', GU.version, GU.getSubDir(runID), '%s.opts' % jobInfo[i][1]))
jobInfo[i] = jobInfo[i] + [runID, outtag, firstEvent, nEvents, False]
# submit jobs
if options.submit:
cmds = []
for job in jobInfo:
singleSize = job[5]/options.nJobs
optSizes = [[job[4] + i*singleSize, singleSize] for i in range(options.nJobs)]
dest='prop',
help='enable prop tube alignment',
default=False)
parser.add_option('--debug',
action='store_true',
dest='debug',
help='enable massive debugging output',
default=False)
(options, args) = parser.parse_args()
print options
if len(sys.argv) < 2:
parser.parse_args(['--help'])
# initialize grid credetial
GU.gridInit()
# prepare the optimized submission sizes
rawFile = 'run_%d_raw.root' % options.run
optSizes = GU.getOptimizedSize(rawFile, options.split, 500)
nJobs = len(optSizes)
if nJobs < 1:
sys.exit()
conf = GU.JobConfig(options.config)
for i in range(options.initial, options.nIter + 1):
print 'Working on alignment cycle', i
# build the raw file
inputFile = rawFile
#runCmd('./update ac %s %s' % (rawFile, inputFile))
dest='port',
help='MySQL port',
default=3283)
parser.add_option('-m',
'--max',
type='int',
dest='nJobsMax',
help='maximum number of uploading instance',
default=8)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
# initialize general tracking configuration
tconf = GU.JobConfig(options.tconfig)
vconf = GU.JobConfig(options.vconfig)
# the uploader
uploader = os.path.join(os.getenv('KTRACKER_ROOT'), 'sqlResWriter')
# initialize the runlist
runIDs = [int(line.strip()) for line in open(options.list).readlines()]
# initialize the history record
trackedRuns = []
vertexedRuns = []
uploadedRuns = []
if options.record == '':
options.record = os.path.join(
GU.workDir, 'record_%s.log' %
dest='mcmode',
help='Running for MC data',
default=False)
parser.add_option('-d',
'--debug',
action='store_true',
dest='debug',
help='Enable massive debugging output',
default=False)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
# initialize grid credetial
GridUtil.gridInit()
# if in catchup mode, then simply submit jobs and exit
if options.resubmit != '':
cmds = [line.strip() for line in open(options.resubmit).readlines()]
GridUtil.submitAllJobs(cmds, options.errlog)
GridUtil.stopGridGuard()
sys.exit(0)
# initialize the configuration
conf = GridUtil.JobConfig(options.config)
# process runID list, if in MC mode, initialize file list from options.list
runIDs = []
runFiles = []
if not options.mcmode: # working with real data, runid is meaningful and is used to find the files
type='string',
dest='job',
help='type of the job',
default='')
parser.add_option('-d',
'--debug',
action='store_true',
dest='debug',
help='Enable massive debugging output',
default=False)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
conf = GU.JobConfig(options.conf)
# initialize the runlist and file list
runFiles = [line.strip().split()[1] for line in open(options.list)]
runIDs = [int(line.strip().split()[0]) for line in open(options.list)]
for index, runFile in enumerate(runFiles):
nTotalJobs, nFinishedJobs, failedOpts = GU.getJobStatus(
conf, options.job, runIDs[index])
if len(failedOpts) != 0 or nTotalJobs != nFinishedJobs:
print 'Certain job failed for ', runIDs[index], runFile
targetFile = os.path.join(conf.indir, runFile)
sourceFiles = [
os.path.join(
import GridUtil as GU
# parse all the commandline controls
parser = OptionParser('Usage: %prog [options]')
parser.add_option('-l', '--list', type = 'string', dest = 'list', help = 'List of run IDs', default = '')
parser.add_option('-c', '--conf', type = 'string', dest = 'conf', help = 'Configuration file', default = '')
parser.add_option('-j', '--job', type = 'string', dest = 'job', help = 'type of the job', default = '')
parser.add_option('-s', '--submit', action = 'store_true', dest = 'submit', help = 'submit the missing jobs', default = False)
parser.add_option('-d', '--debug', action = 'store_true', dest = 'debug', help = 'Enable massive debugging output', default = False)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
conf = GU.JobConfig(options.conf)
# initialize the runlist and file list
runFiles = [line.strip().split()[1] for line in open(options.list)]
runIDs = [int(line.strip().split()[0]) for line in open(options.list)]
cmds = []
for index, runFile in enumerate(runFiles):
nTotalJobs, nFinishedJobs, failedOpts, missingOpts = GU.getJobStatus(conf, options.job, runIDs[index])
if len(failedOpts) != 0 or len(missingOpts) != 0 or nTotalJobs != nFinishedJobs:
print 'Certain job failed for ', runIDs[index], runFile, nTotalJobs, nFinishedJobs, failedOpts, missingOpts
for opt in failedOpts+missingOpts:
cmd = GU.makeCommandFromOpts(options.job, opts, conf) + ' --input=%s' % os.path.join(conf.indir, runFile)
cmds.append(cmd)
continue
parser.add_option('-l', '--list', type = 'string', dest = 'list', help = 'List of run IDs', default = '')
parser.add_option('-j', '--job', type = 'string', dest = 'job', help = 'Type of job: track, vertex, online, etc.', default = '')
parser.add_option('-s', '--split', type = 'int', dest = 'nEvtMax', help = 'Constrain the single job to be less than certain events, default is no splitting', default = -1)
parser.add_option('-c', '--config', type = 'string', dest = 'config', help = 'I/O configuration file', default = '')
parser.add_option('-n', '--maxjobs', type = 'int', dest = 'nJobsMax', help = 'Maximum number of jobs to split one run', default = 10)
parser.add_option('-r', '--resubmit', type = 'string', dest = 'resubmit', help = 'Catchup where it has been left', default = '')
parser.add_option('-e', '--errlog', type = 'string', dest = 'errlog', help = 'Failed command log', default = 'submitAll_err.log')
parser.add_option('-m', '--mcmode', action = 'store_true', dest = 'mcmode', help = 'Running for MC data', default = False)
parser.add_option('-d', '--debug', action = 'store_true', dest = 'debug', help = 'Enable massive debugging output', default = False)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
# initialize grid credetial
GridUtil.gridInit()
# if in catchup mode, then simply submit jobs and exit
if options.resubmit != '':
cmds = [line.strip() for line in open(options.resubmit).readlines()]
GridUtil.submitAllJobs(cmds, options.errlog)
GridUtil.stopGridGuard()
sys.exit(0)
# initialize the configuration
conf = GridUtil.JobConfig(options.config)
# process runID list, if in MC mode, initialize file list from options.list
runIDs = []
runFiles = []
runLengths = []
dest='errlog',
help='Failed command log',
default='trackingMonitor_err.log')
parser.add_option('-d',
'--debug',
action='store_true',
dest='debug',
help='Enable massive debugging output',
default=False)
(options, args) = parser.parse_args()
if len(sys.argv) < 2:
parser.parse_args(['--help'])
# initialize general tracking configuration
tconf = GridUtil.JobConfig(options.tconfig)
vconf = GridUtil.JobConfig(options.vconfig)
# initialize the runlist
runIDs = [int(line.strip()) for line in open(options.list).readlines()]
# initiaize grid
GridUtil.gridInit()
# loop until all tracking jobs on list are done
nExist = 0
fout = open(GridUtil.workDir + '/' + options.errlog, 'w')
while nExist != len(runIDs):
failedJobs = []
vertexJobs = []
nExist = 0