def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
c = shapesim.read_config(run)
cs = shapesim.read_config(c['sim'])
group=options.group
if group is not None:
group = 'group: ['+group+']'
else:
group=''
mode=''
if options.bynode:
mode='mode: bynode'
extra=''
if options.extra is not None:
extra='\n'.join( options.extra.split(';') )
wqd = shapesim.get_wq_dir(run, combine=True)
if not os.path.exists(wqd):
os.makedirs(wqd)
if run[0:8] == 'gmix-fit':
rstr=run.replace('gmix-fit','gmix')
else:
rstr=run
n1 = cs['nums2']
runtype = c['runtype']
if runtype == 'byellip':
n2 = cs['nume']
else:
n2 = shapesim.get_nums2n(c)
for i1 in xrange(n1):
for i2 in xrange(n2):
job_name='%s-combine-%03i-%03i' % (rstr,i1,i2)
wqurl = shapesim.get_wq_url(run,i1,i2,combine=True)
wlog("writing wq script:",wqurl)
with open(wqurl,'w') as fobj:
wqscript=_wqtemplate % {'job_name':job_name,
'run':run,
'i1':i1,
'i2':i2,
'group':group,
'mode':mode,
'extra':extra,
'pri':options.priority}
fobj.write(wqscript)
def main():
options, args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
simname = args[0]
ss = shapesim.ShapeSim(simname)
orient = ss.get("orient", "rand")
if orient == "ring":
numper_str = ""
else:
if len(args) < 2:
parser.print_help()
sys.exit(45)
numper_str = args[1]
if ss.fs != "hdfs":
raise ValueError("This only works for HDFS right now " "would need to worry about making dirs")
c = shapesim.read_config(simname)
groups = options.groups
if groups is None:
groups = ""
else:
groups = "group: [%s]" % groups
wqd = shapesim.get_cache_wq_dir(simname)
if not os.path.exists(wqd):
os.makedirs(wqd)
extra = ""
if options.bynode:
extra = "mode: bynode\nN: 1"
elif options.ncores is not None:
ncores = int(options.ncores)
extra = "mode: bycore1\nN: %d" % ncores
for is2 in xrange(c["nums2"]):
for ie in xrange(c["nume"]):
job_name = "%s-%i-%i" % (simname, is2, ie)
wqurl = shapesim.get_cache_wq_url(simname, is2, ie)
wlog("writing wq script:", wqurl)
with open(wqurl, "w") as fobj:
wqscript = _wqtemplate % {
"job_name": job_name,
"simname": simname,
"is2": is2,
"ie": ie,
"numper": numper_str,
"groups": groups,
"extra": extra,
"pri": options.priority,
}
fobj.write(wqscript)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
c = shapesim.read_config(run)
run = c['run']
sim_config = c['sim_config']
mcmc_config = c['mcmc_config']
s2n_vals = c['s2n_vals']
ns2n = len(s2n_vals)
d = shapesim.get_wq_dir(run, bytrial=True, fs='local')
if not os.path.exists(d):
os.makedirs(d)
d = shapesim.get_output_dir(run, sub='bytrial')
if not os.path.exists(d):
os.makedirs(d)
groups=''
if options.groups is not None:
groups = 'group: [%s]' % options.groups
smax=numpy.iinfo('i8').max
for is2n in xrange(ns2n):
s2n = s2n_vals[is2n]
npair, nsplit = get_npair_nsplit(c, is2n)
for isplit in xrange(nsplit):
job_name='%s-%03i-%03i' % (run,is2n,isplit)
# note the wq logs are local
wqurl = shapesim.get_wq_url(run,0,is2n,itrial=isplit,fs='local')
output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
seed = numpy.random.randint(smax)
wlog("writing wq script:",wqurl)
with open(wqurl,'w') as fobj:
d={'job_name':job_name,
'version':options.version,
'groups':groups,
'pri':options.priority,
'sim_config':sim_config,
'mcmc_config':mcmc_config,
's2n':s2n,
'npair':npair,
'seed':seed,
'output':output}
wqscript=_wqtemplate % d
fobj.write(wqscript)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
if options.skip is None:
skip=[]
else:
skip = [int(v) for v in options.skip.split(',')]
c = shapesim.read_config(run)
s2n_vals = c['s2n_vals']
n_s2n = len(s2n_vals)
dlist=[]
#for is2n in reversed(xrange(n_s2n)):
for is2n in xrange(n_s2n):
if is2n in skip:
continue
s2n_matched = s2n_vals[is2n]
fname=shapesim.get_output_url(run, 0, is2n)
print >>stderr,fname
data=eu.io.read(fname)
if options.gerror:
d = get_averaged_gerror(data, s2n_matched, verbose=options.verbose)
else:
plot_dir=shapesim.get_plot_dir(run)
plot_name=get_image_file(plot_dir, fname)
d = get_averaged_jackknife(data, s2n_matched,
verbose=options.verbose,
show=options.show,
fname=plot_name)
dlist.append(d)
output = eu.numpy_util.combine_arrlist(dlist)
out_fname=shapesim.get_averaged_url(run, 0)
print >>stderr,'writing:',out_fname
eu.io.write(out_fname, output, clobber=True)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
max_jobs=int(options.max_jobs)
c = shapesim.read_config(run)
c['version'] = options.version
make_some_dirs(run)
master_script=write_master(c)
write_condor_file(c, master_script, missing=options.missing,
max_jobs=max_jobs)
def main():
options, args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run = args[0]
c = shapesim.read_config(run)
c["version"] = options.version
if options.wshear:
c["exec_name"] = "gsim-ring-mcmc-wshear"
else:
c["exec_name"] = "gsim-ring-mcmc"
ns2n = len(c["s2n_vals"])
make_some_dirs(run)
master_script = write_master(c)
write_condor_file(c, master_script)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
c = shapesim.read_config(run)
ns2n=len(c['s2n_vals'])
flist=get_flist(run)
if options.is2n is None:
is2n_vals = range(len(c['s2n_vals']))
else:
is2n_vals = options.is2n.split(',')
is2n_vals = [int(ii) for ii in is2n_vals]
for is2n in is2n_vals:
check_is2n(run, is2n, flist, full=options.full)
def check_is2n(run, is2n, flist, full=False):
fs=get_default_fs()
c = read_config(run)
npair,nsplit = get_npair_nsplit(c, is2n)
ntot=npair*2
for isplit in xrange(nsplit):
f=get_output_url(run, 0, is2n, itrial=isplit, fs=fs)
if f not in flist:
print 'missing:',f
continue
if full:
t=None
try:
t=eu.io.read(f)
except:
print 'failed to read:',f
if t is not None:
if t.size != ntot:
print "expected %d, got %d" % (npair,t.size)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 1:
parser.print_help()
sys.exit(45)
run=args[0]
c = shapesim.read_config(run)
cs = shapesim.read_config(c['sim'])
if options.bytrial:
if 'stack-' in run:
ntrial=c['nsplit']
else:
orient=cs.get('orient','rand')
if orient == 'ring':
ntrial = cs['nsplit']
else:
ntrial = cs['ntrial']
wqd = shapesim.get_wq_dir(run, bytrial=options.bytrial)
if not os.path.exists(wqd):
os.makedirs(wqd)
extra=''
if options.bynode:
extra='mode: bynode\nN: 1'
elif options.ncores is not None:
ncores=int(options.ncores)
extra='mode: bycore1\nN: %d' % ncores
if run[0:8] == 'gmix-fit':
rstr=run.replace('gmix-fit','gmix')
else:
rstr=run
n1 = shapesim.get_numT(cs)
runtype = c['runtype']
if runtype == 'byellip':
n2 = cs['nume']
else:
n2 = shapesim.get_nums2n(c)
for i1 in xrange(n1):
for i2 in xrange(n2):
groups=''
if options.i2new is not None and i2 <= int(options.i2new):
groups='group: [new,new2]'
elif options.i2new1 is not None and i2 <= int(options.i2new1):
groups='group: [new]'
elif options.i2new2 is not None and i2 <= int(options.i2new2):
groups='group: [new2]'
elif options.groups is not None:
groups = 'group: [%s]' % options.groups
if options.bytrial:
for itrial in xrange(ntrial):
job_name='%s-%03i-%03i-%02i' % (rstr,i1,i2,itrial)
wqurl = shapesim.get_wq_url(run,i1,i2,itrial=itrial)
wlog("writing wq script:",wqurl)
with open(wqurl,'w') as fobj:
d={'job_name':job_name,
'run':run,
'i1':i1,
'i2':i2,
'itrial':itrial,
'groups':groups,
'extra':extra,
'pri':options.priority}
wqscript=_wqtemplate_bytrial % d
fobj.write(wqscript)
else:
job_name='%s-%03i-%03i' % (rstr,i1,i2)
wqurl = shapesim.get_wq_url(run,i1,i2)
wlog("writing wq script:",wqurl)
with open(wqurl,'w') as fobj:
wqscript=_wqtemplate % {'job_name':job_name,
'run':run,
'i1':i1,
'i2':i2,
'groups':groups,
'extra':extra,
'pri':options.priority}
fobj.write(wqscript)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 2:
parser.print_help()
sys.exit(1)
sim_name=args[0]
run_type=args[1]
cs = shapesim.read_config(sim_name)
objmodel = cs['objmodel']
psfmodel = cs['psfmodel']
simtype = shortnames[objmodel]+shortnames[psfmodel]
if options.run_name is not None:
run_name=options.run_name
fname = shapesim.get_config_file(run_name)
if os.path.exists(fname) and not options.overwrite:
raise ValueError("file already exists: %s" % fname)
else:
run_name, fname = get_run_name_and_file(sim_name)
s2n_fac=options.s2n_fac
if s2n_fac is None:
raise ValueError("send --s2n-fac")
if run_type == 'bys2n':
ie=options.ie
if ie is None:
raise ValueError("send --ie for bys2n")
tmps2,e_val = shapesim.get_s2_e(cs, 0, int(ie))
#s2n_vals_def='5,10,15,20,25,30,40,50,60,70,80,90,100'
s2n_vals_def='10,15,20,25,30,40,50,60,70,80,90,100'
s2n_vals=options.s2n_vals
if s2n_vals is None:
s2n_vals=s2n_vals_def
if simtype == 'edg':
text=edg_bys2n_template % {'run_name':run_name,
'sim_name':sim_name,
's2n_fac':s2n_fac,
's2n_vals':s2n_vals,
'ie':ie,
'e_val':e_val}
elif simtype == 'et':
text=et_bys2n_template % {'run_name':run_name,
'sim_name':sim_name,
's2n_fac':s2n_fac,
's2n_vals':s2n_vals,
'ie':ie,
'e_val':e_val}
elif simtype == 'dt':
text=dt_bys2n_template % {'run_name':run_name,
'sim_name':sim_name,
's2n_fac':s2n_fac,
's2n_vals':s2n_vals,
'ie':ie,
'e_val':e_val}
elif simtype == 'gg':
text=gg_bys2n_template % {'run_name':run_name,
'sim_name':sim_name,
's2n_fac':s2n_fac,
's2n_vals':s2n_vals,
'ie':ie,
'e_val':e_val}
else:
raise ValueError("support other sim types")
elif run_type == 'byellip':
s2n=options.s2n
if s2n is None:
raise ValueError("send s2n for byellip")
if simtype == 'edg':
text=edg_byellip_template
elif simtype == 'gg':
text=gg_byellip_template
else:
raise ValueError("support other sim types")
text=text % {'run_name':run_name,
'sim_name':sim_name,
's2n_fac':s2n_fac,
's2n':s2n}
else:
raise ValueError("support others?")
print text
print 'fname:',fname
if not options.dryrun:
with open(fname,'w') as fobj:
fobj.write(text)
else:
print 'dryrun'
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 3:
parser.print_help()
sys.exit(45)
run=args[0]
nodes=int(args[1])
ppn=int(args[2])
np=nodes*ppn
# seconds per ellip ring pair. this is for exp...
seconds_per=get_seconds_per_pair()
c = shapesim.read_config(run)
sim_config = c['sim_config']
mcmc_config = c['mcmc_config']
nsplit = c['nsplit']
s2n_vals = c['s2n_vals']
s2n_fac = c['s2n_fac']
min_npair = c['min_npair']
ns2n = len(s2n_vals)
pbsd = shapesim.get_pbs_dir(run)
logdir=os.path.join(pbsd,'logs')
outd = shapesim.get_output_dir(run, sub='bytrial')
if not os.path.exists(pbsd):
os.makedirs(pbsd)
if not os.path.exists(logdir):
os.makedirs(logdir)
if not os.path.exists(outd):
os.makedirs(outd)
rstr=run
job_name = '-'.join( (rstr.split('-'))[1:] )
script_url=shapesim.get_minions_script_url(run)
script_base=os.path.basename(script_url)
print script_url
with open(script_url,'w') as fobj:
c['version'] = options.version
fobj.write(_script_template % c)
print
pbsf=shapesim.get_minions_url(run,0)
cmdf=shapesim.get_commands_url(run,0)
print cmdf
ntot=0
with open(cmdf,'w') as fobj:
for is2n in xrange(ns2n):
s2n = s2n_vals[is2n]
npair = shapesim.get_s2n_nrepeat(s2n, fac=s2n_fac)
if npair < min_npair:
npair = min_npair
ntot += npair*nsplit
for isplit in xrange(nsplit):
output = shapesim.get_output_url(run, 0, is2n, itrial=isplit)
outbname=os.path.basename(output)
logf=outbname.replace('.rec','.log')
cmd=_cmd_template.format(script_file=script_base,
s2n=s2n,
npair=npair,
output=output,
logfile=logf)
fobj.write(cmd)
fobj.write('\n')
time_seconds = ntot*seconds_per/(np-1)
print pbsf
hours_raw = time_seconds/3600.
hours = int(round(hours_raw)) + 1
print ' raw hours:',hours_raw,'hours used:',hours
if options.version=='hopper':
minions_cmd='minions-hopper'
else:
minions_cmd='minions'
with open(pbsf,'w') as fobj:
cmd_bname=os.path.basename(cmdf)
pbslog=os.path.basename(pbsf)+'.pbslog'
pbs_text=_pbs_template.format(minions_cmd=minions_cmd,
job_name=job_name,
pbslog=pbslog,
nodes=nodes,
ppn=ppn,
np=np,
hours=hours,
commands_file=cmd_bname)
fobj.write(pbs_text)
def main():
options,args = parser.parse_args(sys.argv[1:])
if len(args) < 2:
parser.print_help()
sys.exit(45)
run=args[0]
nodes=int(args[1])
ppn=int(options.ppn)
np=nodes*ppn
# seconds per ellip ring pair. this is for exp...
c = shapesim.read_config(run)
cs = shapesim.read_config(c['sim'])
seconds_per=get_seconds_per(run,c)
nsplit = cs['nsplit']
pbsd = shapesim.get_pbs_dir(run)
logdir=os.path.join(pbsd,'logs')
if not os.path.exists(pbsd):
os.makedirs(pbsd)
if not os.path.exists(logdir):
os.makedirs(logdir)
if run[0:8] == 'gmix-fit':
rstr=run.replace('gmix-fit','gmix')
else:
rstr=run
job_name = '-'.join( (rstr.split('-'))[1:] )
if 'nums2' in cs:
n1 = cs['nums2']
else:
n1 = len(cs['Tobj'])
n2 = shapesim.get_nums2n(c)
script_url=shapesim.get_minions_script_url(run)
script_base=os.path.basename(script_url)
print script_url
with open(script_url,'w') as fobj:
fobj.write(_script_template)
print
for i1 in xrange(n1):
pbsf=shapesim.get_minions_url(run,i1)
cmdf=shapesim.get_commands_url(run,i1)
print cmdf
ntot=0
with open(cmdf,'w') as fobj:
for i2 in xrange(n2):
nellip=get_nellip(c,i2)
for itrial in xrange(nsplit):
outf=shapesim.get_output_url(run, i1, i2, itrial=itrial)
outbname=os.path.basename(outf)
logf=outbname.replace('.rec','.log')
cmd=_cmd_template.format(script_file=script_base,
run=run,
i1=i1,
i2=i2,
itrial=itrial,
logfile=logf)
fobj.write(cmd)
fobj.write('\n')
ntot += nellip
time_seconds = ntot*seconds_per/(np-1)
print pbsf
hours_raw = time_seconds/3600.
hours = int(round(hours_raw)) + 1
print ' raw hours:',hours_raw,'hours used:',hours
with open(pbsf,'w') as fobj:
cmd_bname=os.path.basename(cmdf)
pbslog=os.path.basename(pbsf)+'.pbslog'
pbs_text=_pbs_template.format(job_name=job_name,
pbslog=pbslog,
nodes=nodes,
ppn=ppn,
np=np,
hours=hours,
commands_file=cmd_bname)
fobj.write(pbs_text)
