os.remove(outFile)
cmd = 'python %s -o %s %s' % (hdfMerge, tempOut, tempFiles)
if not args.test:
ex = [
'mkdir -p %s' % tempDir,
'cp %s %s' % (files, tempDir),
'%s %s' % (my.env, cmd),
'rm -f %s' % tempFiles,
'mv %s %s' % (tempOut, prefix)
]
else:
ex = [cmd]
exList += [ex]
jobIDs += ['llhmerge_%s_%s' % (args.config, date)]
if args.test:
exList = exList[:2]
njobs = len(exList)
if njobs > 500:
yn = raw_input('Submit %i jobs? [y|n]: ' % njobs)
if yn != 'y':
raise SystemExit('Aborting...')
print 'Submitting %i batches...' % njobs
for ex, jobID in zip(exList, jobIDs):
py_submit(ex, test=args.test, jobID=jobID)
outp = "${_CONDOR_SCRATCH_DIR}/simple-dst"
out = '%s/ic86_%s.root' % (outp, jobID)
lfiles = ['%s/%s' % (outp, os.path.basename(f)) for f in batch]
lfiles = ' '.join(lfiles)
cmd = ' '.join([my.env, 'dstcut', out, lfiles])
ex = [
'mkdir -p %s' % outp,
'cp %s %s' % (files, outp), # Copy files for local I/O
#'%s' % my.env, # Load environment shell
'%s' % cmd, # Process files
'rm -f %s' % lfiles, # Remove copied ROOT files
'mv %s %s' % (out, outdir) # Move output file
]
exs += [ex]
#out = '%s/ic86_%s.root' % (outdir, jobID)
#exs += [[' '.join([my.env, 'dstcut', out, files])]]
jobIDs += [jobID]
print 'Submitting %i jobs...' % len(exs)
if len(exs) > 500:
raise SystemExit('Too many jobs. Use larger batches.')
# Submit
cwd = os.getcwd()
os.chdir(my.npx4)
for ex, jobID in zip(exs, jobIDs):
py_submit(ex, jobID=jobID, test=args.test)
os.chdir(cwd)
c_opts += ' --sbins %s' % ' '.join(args.sbins)
if args.comp:
print ' --comp ' + ' '.join(args.comp)
c_opts += ' --comp %s' % ' '.join(args.comp)
# Increase requested memory for jobs that produce multiple maps
sublines = None
if args.ebins or args.sbins or args.comp:
sublines = ["request_memory = 5000"]
# Submit files
cwd = os.getcwd()
os.chdir(my.npx4)
print('Submitting {} files...'.format(len(batchList)))
for i in range(len(batchList)):
jobID = 'timescramble_{}_{:05d}'.format(args.config, i)
cmd = '{}/timeScramble'.format(cwd)
ex = [cmd, '--batch_idx', str(i), c_opts]
if not args.test:
ex = [my.env] + ex
#print('ex = {}'.format(ex))
ex = [' '.join(ex)]
#print('ex = {}'.format(ex))
py_submit(ex, sublines=sublines, test=args.test, jobID=jobID)
os.chdir(cwd)
end = re.split('\.', batch[-1])[-3]
outFile = '%s/CountTable_%s_%s' % (outDir, sim, args.bintype)
outFile += '_Part'+start+'-'+end+'.npy'
batch.insert(0, gcd)
batch = ' '.join(batch)
cmd = 'python %s/MakeHist.py' % cwd
ex = '%s -f %s -b %s -o %s' % (cmd, batch, args.bintype, outFile)
if not args.test:
ex = ' '.join([my.env, ex])
exList += [[ex]]
# Moderate number of submitted jobs
njobs = len(exList)
if njobs > 500:
yn = raw_input('About to submit %i jobs. You sure? [y|n]: ' % njobs)
if yn != 'y':
raise SystemExit('Aborting...' % njobs)
# Submit jobs
print 'Submitting %i batches...' % njobs
for ex in exList:
py_submit(ex, test=args.test)