def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
c = shapesim.read_config(run)
run = c['run']
sim_config = c['sim_config']
mcmc_config = c['mcmc_config']
s2n_vals = c['s2n_vals']
ns2n = len(s2n_vals)
d = shapesim.get_wq_dir(run, bytrial=True, fs='local')
if not os.path.exists(d):
os.makedirs(d)
d = shapesim.get_output_dir(run, sub='bytrial')
if not os.path.exists(d):
os.makedirs(d)
groups=''
if options.groups is not None:
groups = 'group: [%s]' % options.groups
smax=numpy.iinfo('i8').max
for is2n in xrange(ns2n):
s2n = s2n_vals[is2n]
npair, nsplit = get_npair_nsplit(c, is2n)
for isplit in xrange(nsplit):
job_name='%s-%03i-%03i' % (run,is2n,isplit)
# note the wq logs are local
wqurl = shapesim.get_wq_url(run,0,is2n,itrial=isplit,fs='local')
output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
seed = numpy.random.randint(smax)
wlog("writing wq script:",wqurl)
with open(wqurl,'w') as fobj:
d={'job_name':job_name,
'version':options.version,
'groups':groups,
'pri':options.priority,
'sim_config':sim_config,
'mcmc_config':mcmc_config,
's2n':s2n,
'npair':npair,
'seed':seed,
'output':output}
wqscript=_wqtemplate % d
fobj.write(wqscript)
def write_condor_file(c, master_script, equal_time=False):
run = c["run"]
job_name = "-".join((run.split("-"))[1:])
condor_job_url = shapesim.get_condor_job_url(run)
ns2n = len(c["s2n_vals"])
seconds_per = get_seconds_per_pair()
if "desired_err" in c:
do_by_noise = True
else:
do_by_noise = False
print condor_job_url
njobs = 0
smax = numpy.iinfo("i8").max
with open(condor_job_url, "w") as fobj:
text = _condor_template_head.format(master_script=master_script, overall_name=job_name)
fobj.write(text)
# for is2n in reversed(xrange(ns2n)):
for is2n in xrange(ns2n):
s2n = c["s2n_vals"][is2n]
npair, nsplit = get_npair_nsplit(c, is2n)
time_hours = npair * seconds_per / 3600.0
if time_hours > MAXTIME_HOURS:
raise ValueError(
"time is greater than %.2f "
"hours: %d*%.2f/3600.0 = %s" % (MAXTIME_HOURS, npair, seconds_per, time_hours)
)
print "nsplit:", nsplit, "npair:", npair, "time (hours):", time_hours
njobs += nsplit
for isplit in xrange(nsplit):
output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
logfile = output.replace(".rec", ".log")
this_job_name = "%s-%03d-%03d" % (job_name, is2n, isplit)
# seed = numpy.random.randint(smax)
qdata = _queue_template.format(
job_name=this_job_name,
s2n=s2n,
npair=npair,
# seed=seed,
output=output,
logfile=logfile,
)
fobj.write(qdata)
print "total jobs:", njobs
def get_flist(run):
import glob
fs=shapesim.get_default_fs()
f=shapesim.get_output_url(run, 0, 0, itrial=0, fs=fs)
d=os.path.dirname(f)
if 'ngmix' in run:
flist=glob.glob(d+'/*.fits')
else:
flist=glob.glob(d+'/*.rec')
return flist
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
if options.skip is None:
skip=[]
else:
skip = [int(v) for v in options.skip.split(',')]
c = shapesim.read_config(run)
s2n_vals = c['s2n_vals']
n_s2n = len(s2n_vals)
dlist=[]
#for is2n in reversed(xrange(n_s2n)):
for is2n in xrange(n_s2n):
if is2n in skip:
continue
s2n_matched = s2n_vals[is2n]
fname=shapesim.get_output_url(run, 0, is2n)
print >>stderr,fname
data=eu.io.read(fname)
if options.gerror:
d = get_averaged_gerror(data, s2n_matched, verbose=options.verbose)
else:
plot_dir=shapesim.get_plot_dir(run)
plot_name=get_image_file(plot_dir, fname)
d = get_averaged_jackknife(data, s2n_matched,
verbose=options.verbose,
show=options.show,
fname=plot_name)
dlist.append(d)
output = eu.numpy_util.combine_arrlist(dlist)
out_fname=shapesim.get_averaged_url(run, 0)
print >>stderr,'writing:',out_fname
eu.io.write(out_fname, output, clobber=True)
def check_is2n(run, is2n, flist, full=False):
fs=get_default_fs()
c = read_config(run)
npair,nsplit = get_npair_nsplit(c, is2n)
ntot=npair*2
for isplit in xrange(nsplit):
f=get_output_url(run, 0, is2n, itrial=isplit, fs=fs)
if f not in flist:
print 'missing:',f
continue
if full:
t=None
try:
t=eu.io.read(f)
except:
print 'failed to read:',f
if t is not None:
if t.size != ntot:
print "expected %d, got %d" % (npair,t.size)
def write_condor_file(c, master_script, equal_time=False, missing=False, max_jobs=10000):
run=c['run']
overall_name = '-'.join( (run.split('-'))[1:] )
if missing:
flist=get_flist(run)
overall_name += 'missing'
else:
flist=None
ns2n=len(c['s2n_vals'])
seconds_per=get_seconds_per_pair(c)
print 'seconds per:',seconds_per
if 'desired_err' in c:
do_by_noise=True
else:
do_by_noise=False
njobs_thisfile=0
njobs=0
old_filenum=-1
filenum=0
smax=numpy.iinfo('i8').max
fobj=start_new_file(run, filenum, master_script,
overall_name, missing=missing)
for is2n in xrange(ns2n):
s2n=c['s2n_vals'][is2n]
npair, nsplit = get_npair_nsplit(c, is2n)
time_hours = npair*seconds_per/3600.0
if time_hours > MAXTIME_HOURS:
raise ValueError("time is greater than %.2f "
"hours: %d*%.2f/3600.0 = %s" % (MAXTIME_HOURS,npair,seconds_per,time_hours))
print ' nsplit:',nsplit,'npair:',npair,'time (hours):',time_hours
for isplit in xrange(nsplit):
output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
logfile = output.replace('.fits','.log')
this_job_name='%s-%03d-%05d' % (overall_name,is2n,isplit)
qdata=_queue_template.format(job_name=this_job_name,
s2n=s2n,
npair=npair,
output=output,
logfile=logfile)
do_write=True
if missing and output in flist:
do_write=False
if do_write:
njobs += 1
njobs_thisfile += 1
fobj.write(qdata)
if njobs_thisfile >= max_jobs:
filenum += 1
njobs_thisfile=0
fobj.close()
fobj=start_new_file(run, filenum, master_script,
overall_name, missing=missing)
print 'total jobs:',njobs
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 3:
parser.print_help()
sys.exit(45)
run=args[0]
nodes=int(args[1])
ppn=int(args[2])
np=nodes*ppn
# seconds per ellip ring pair. this is for exp...
seconds_per=get_seconds_per_pair()
c = shapesim.read_config(run)
sim_config = c['sim_config']
mcmc_config = c['mcmc_config']
nsplit = c['nsplit']
s2n_vals = c['s2n_vals']
s2n_fac = c['s2n_fac']
min_npair = c['min_npair']
ns2n = len(s2n_vals)
pbsd = shapesim.get_pbs_dir(run)
logdir=os.path.join(pbsd,'logs')
outd = shapesim.get_output_dir(run, sub='bytrial')
if not os.path.exists(pbsd):
os.makedirs(pbsd)
if not os.path.exists(logdir):
os.makedirs(logdir)
if not os.path.exists(outd):
os.makedirs(outd)
rstr=run
job_name = '-'.join( (rstr.split('-'))[1:] )
script_url=shapesim.get_minions_script_url(run)
script_base=os.path.basename(script_url)
print script_url
with open(script_url,'w') as fobj:
c['version'] = options.version
fobj.write(_script_template % c)
print
pbsf=shapesim.get_minions_url(run,0)
cmdf=shapesim.get_commands_url(run,0)
print cmdf
ntot=0
with open(cmdf,'w') as fobj:
for is2n in xrange(ns2n):
s2n = s2n_vals[is2n]
npair = shapesim.get_s2n_nrepeat(s2n, fac=s2n_fac)
if npair < min_npair:
npair = min_npair
ntot += npair*nsplit
for isplit in xrange(nsplit):
output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
outbname=os.path.basename(output)
logf=outbname.replace('.rec','.log')
cmd=_cmd_template.format(script_file=script_base,
s2n=s2n,
npair=npair,
output=output,
logfile=logf)
fobj.write(cmd)
fobj.write('\n')
time_seconds = ntot*seconds_per/(np-1)
print pbsf
hours_raw = time_seconds/3600.
hours = int(round(hours_raw)) + 1
print ' raw hours:',hours_raw,'hours used:',hours
if options.version=='hopper':
minions_cmd='minions-hopper'
else:
minions_cmd='minions'
with open(pbsf,'w') as fobj:
cmd_bname=os.path.basename(cmdf)
pbslog=os.path.basename(pbsf)+'.pbslog'
pbs_text=_pbs_template.format(minions_cmd=minions_cmd,
job_name=job_name,
pbslog=pbslog,
nodes=nodes,
ppn=ppn,
np=np,
hours=hours,
commands_file=cmd_bname)
fobj.write(pbs_text)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 2:
parser.print_help()
sys.exit(45)
run=args[0]
nodes=int(args[1])
ppn=int(options.ppn)
np=nodes*ppn
# seconds per ellip ring pair. this is for exp...
c = shapesim.read_config(run)
cs = shapesim.read_config(c['sim'])
seconds_per=get_seconds_per(run,c)
nsplit = cs['nsplit']
pbsd = shapesim.get_pbs_dir(run)
logdir=os.path.join(pbsd,'logs')
if not os.path.exists(pbsd):
os.makedirs(pbsd)
if not os.path.exists(logdir):
os.makedirs(logdir)
if run[0:8] == 'gmix-fit':
rstr=run.replace('gmix-fit','gmix')
else:
rstr=run
job_name = '-'.join( (rstr.split('-'))[1:] )
if 'nums2' in cs:
n1 = cs['nums2']
else:
n1 = len(cs['Tobj'])
n2 = shapesim.get_nums2n(c)
script_url=shapesim.get_minions_script_url(run)
script_base=os.path.basename(script_url)
print script_url
with open(script_url,'w') as fobj:
fobj.write(_script_template)
print
for i1 in xrange(n1):
pbsf=shapesim.get_minions_url(run,i1)
cmdf=shapesim.get_commands_url(run,i1)
print cmdf
ntot=0
with open(cmdf,'w') as fobj:
for i2 in xrange(n2):
nellip=get_nellip(c,i2)
for itrial in xrange(nsplit):
outf=shapesim.get_output_url(run, i1, i2, itrial=itrial)
outbname=os.path.basename(outf)
logf=outbname.replace('.rec','.log')
cmd=_cmd_template.format(script_file=script_base,
run=run,
i1=i1,
i2=i2,
itrial=itrial,
logfile=logf)
fobj.write(cmd)
fobj.write('\n')
ntot += nellip
time_seconds = ntot*seconds_per/(np-1)
print pbsf
hours_raw = time_seconds/3600.
hours = int(round(hours_raw)) + 1
print ' raw hours:',hours_raw,'hours used:',hours
with open(pbsf,'w') as fobj:
cmd_bname=os.path.basename(cmdf)
pbslog=os.path.basename(pbsf)+'.pbslog'
pbs_text=_pbs_template.format(job_name=job_name,
pbslog=pbslog,
nodes=nodes,
ppn=ppn,
np=np,
hours=hours,
commands_file=cmd_bname)
fobj.write(pbs_text)
