alltracts = LR.organize_items(tracts, njobs)
# Loop over filters
for filt in opts.filters:
for i, tracts in enumerate(alltracts):
cmd = ""
for tract in tracts:
newfile = open('%s_%s.list' % (filt, str(tract)), 'w')
for visit in tracts_visits[tract][filt]:
newfile.write('--id tract=%s visit=%s\n' %
(str(tract), str(visit)))
newfile.close()
cmd += "jointcal.py %s --output %s @%s_%s.list --configfile %s --clobber-versions -L DEBUG\n" % \
(input, output, filt, str(tract), config)
# Only submit the job if asked
prefix = "jointcal_%s_%03d" % (filt, i + 1)
LR.submit(cmd,
prefix,
filt,
autosubmit=opts.autosubmit,
ct=opts.ct,
vmem=opts.vmem,
queue=opts.queue,
system=opts.system,
otheroptions=opts.otheroptions,
from_slac=opts.fromslac)
if not opts.autosubmit:
print("\nINFO: Use option --autosubmit to submit the jobs")
usage = """%prog [option]"""
description = """This script will run forcedPhotCoadd for a given list of filters and patches.
The default if to use f.list files (where 'f' is a filter in ugriz) and patches_f.txt,
and launch forcedPhotCoadd in several batch jobs. You thus need to be running it at CC-IN2P3
to make it work. To run all filters, you can do something like
%prog -f ugriz -m 1 -c YOURCONFIGFILE -a"""
opts, args = LR.standard_options(usage=usage, description=description)
#opts.input = "pardir/output"
#opts.output = "pardir/output"
for filt in opts.filters:
filePatch = "patches_" + filt + ".txt"
cmd = "split -a 5 -l " + str(
opts.mod) + " -d " + filePatch + " " + filePatch + "_"
os.system(cmd)
patchList = glob.glob(filePatch + "_*")
print("\nWorning on filter %s: %i patches (jobs)" %
(filt, len(patchList)))
for patch in sorted(patchList):
cmd = build_cmd(patch, opts.configs, filt, opts.input, opts.output)
LR.submit(cmd,
patch,
filt,
autosubmit=opts.autosubmit,
from_slac=opts.fromslac)
print("WARNING: No file (no visit) for filter", filt)
continue
# Get the list of patches
patches = [" ".join(p) for p in N.loadtxt("patches_" + filt + ".txt",
dtype='bytes').astype('str')]
print("INFO: %i patches loaded: " % len(patches))
# How many jobs should we be running (and how many visit in each?)?
njobs = LR.job_number(patches, opts.mod, opts.max)
# Reorganize the visit list in consequence
patches = LR.organize_items(patches, njobs)
# Loop over the patches sub lists
for i, ps in enumerate(patches):
# Build the command line and other things
cmd = build_cmd("@scripts/%s/patches_%03d.list" % (filt, i + 1),
config, filt, opts.input, opts.output)
# Only submit the job if asked
LR.submit(cmd, "patches_%03d" % (i + 1), filt, autosubmit=opts.autosubmit,
ct=5000, vmem='4G', from_slac=opts.fromslac)
N.savetxt("scripts/%s/patches_%03d.list" % (filt, i + 1),
N.array(ps, dtype='str').tolist(), fmt="%s")
if not opts.autosubmit:
print("\nINFO: Use option --autosubmit to submit the jobs")
njobs = LR.job_number(patches, opts.mod, opts.max)
# Reorganize the visit list in consequence
patches = LR.organize_items(patches, njobs)
# Loop over the patches sub lists
for i, ps in enumerate(patches):
# Build the command line and other things
cmd = build_cmd("@scripts/%s/patches_%03d.list" % (filt, i + 1),
config, filt, opts.input, opts.output)
# Only submit the job if asked
prefix = "patches_%03d" % (i + 1)
LR.submit(cmd,
prefix,
filt,
autosubmit=opts.autosubmit,
queue=opts.queue,
ct=6000,
vmem=opts.vmem,
system=opts.system,
from_slac=opts.fromslac)
N.savetxt("scripts/%s/patches_%03d.list" % (filt, i + 1),
N.array(ps, dtype='str').tolist(),
fmt="%s")
if not opts.autosubmit:
print("\nINFO: Use option --autosubmit to submit the jobs")
and launch detectCoadSources in several batch jobs. You thus need to be running it at CC-IN2P3
to make it work. To run all filters, you can do something like
%prog -f ugriz -m 1 -c detectCoaddSources.py -a"""
opts, args = LR.standard_options(usage=usage, description=description)
# overwrite some options
opts.mod = 2
#opts.input = "pardir/output"
#opts.output = "pardir/output"
file_patch = "patches_all.txt"
cmd = "split -a 4 -l " + str(
opts.mod) + " -d " + file_patch + " " + file_patch + "_"
os.system(cmd)
patch_list = sorted(glob.glob(file_patch + "_*"))
for patch in sorted(patch_list):
print("\n", patch)
# Only submit the job if asked
cmd = build_cmd(patch,
opts.input,
opts.output,
configFile=opts.configs)
LR.submit(cmd,
patch,
autosubmit=opts.autosubmit,
ct=10000,
vmem='4G',
from_slac=opts.fromslac)