def lensout_read_old(file=None, run=None, lens_split=None, silent=False, old=False):
"""
Note old means something different here
"""
if file is None and run is None:
raise ValueError("usage: lensout_read(file=, run=)")
if file is None:
if lens_split is not None:
file = sample_file(type="lensout", sample=run, lens_split=lens_split)
return lensout_read(file=file)
return lensout_read_byrun(run)
if not silent:
stdout.write("Opening lensout file: %s\n" % file)
file = expand_path(file)
fobj = open(file, "r")
if old:
idt = "i4"
else:
idt = "i8"
nlens = numpy.fromfile(fobj, dtype=idt, count=1)
if not silent:
stdout.write(" nlens: %s\n" % nlens[0])
nbin = numpy.fromfile(fobj, dtype=idt, count=1)
if not silent:
stdout.write(" nbin: %s\n" % nbin[0])
dt = lensout_dtype_old(nbin[0], old=old)
if not silent:
stdout.write("Reading with dtype: %s\n" % str(dt))
data = numpy.fromfile(fobj, dtype=dt, count=nlens[0])
fobj.close()
return data
def create_pbs_byrun(self,
proctype,
procrun,
espy_v=None,
esutil_v=None,
stomp_v=None,
photo_sweep=None,
photo_calib=None,
photo_resolve=None):
import sdsspy
import pbs
p = sdsspy.sweeps.Proc(proctype,procrun)
extra=''
if self.primonly:
extra=', primonly=True'
pbsdir=path_join('~/pbs',proctype,procrun)
pbsdir = expand_path(pbsdir)
if not os.path.exists(pbsdir):
stdout.write("Making directory: %s\n" % pbsdir)
os.makedirs(pbsdir)
if espy_v is None:
espy_v = "-r ~esheldon/exports/espy-work"
if esutil_v is None:
esutil_v="-r ~esheldon/exports/esutil-work"
if stomp_v is None:
stomp_v="-r ~esheldon/exports/stomp-work"
if photo_sweep is None:
photo_sweep=os.getenv("PHOTO_SWEEP")
if photo_resolve is None:
photo_resolve=os.getenv("PHOTO_RESOLVE")
if photo_calib is None:
photo_calib=os.getenv("PHOTO_CALIB")
setups = \
[espy_v,esutil_v,stomp_v]
setups = ['setup '+s for s in setups]
setups += ['export PHOTO_SWEEP=%s' % photo_sweep,
'export PHOTO_CALIB=%s' % photo_calib,
'export PHOTO_RESOLVE=%s' % photo_resolve]
qsub_filename=path_join(pbsdir, 'submit-%s-%s-byrun.sh' % (proctype,procrun))
loadcol_filename=path_join(pbsdir, 'loadcol-%s-%s-byrun.py' % (proctype,procrun))
loadcol_file=open(loadcol_filename,'w')
loadcol_file.write("import sdsspy\n")
loadcol_file.write("p=sdsspy.sweeps.Proc('%s','%s')\n" % (proctype,procrun))
loadcol_file.write("p.make_columns()\n")
loadcol_file.close()
qsub_file=open(qsub_filename, 'w')
irun=0
nrun=len(p._runs)
for run in sorted(p._runs):
stdout.write("Creating pbs file for run: %s\n" % run)
# create the pbs file name
rstr=sdsspy.files.run2string(run)
job_name = '%s-%s' % (proctype,rstr)
pbsfilename='%s-%s-%s.pbs' % (proctype, procrun, rstr)
pbsfilename=path_join(pbsdir, pbsfilename)
# the python commands to execute
python_commands = """
import es_sdsspy
import sdssgal
gs=sdssgal.GalSelector
p=es_sdsspy.sweeps.Proc('sdssgal','%s','gal')
p.process_runs(gs, runs=%s%s)""" % (procrun,run,extra)
ppy = pbs.PBSPython(pbsfilename,python_commands,
job_name=job_name, setups=setups)
ppy.write()
# write to the submission file
qsub_file.write('echo -n "%s/%s " \n' % (irun+1,nrun))
qsub_file.write('qsub %s\n' % pbsfilename)
irun += 1
qsub_file.close()
def __init__(self,
filename,
command,
job_name=None,
queue='fast',
walltime=None, # e.g. '24:00:00'
nodes=1,
ppn=1,
setups=None,
no_redirect=False):
filename = expand_path(filename)
self.filename = filename
pbslog=filename+'.pbslog'
logf = filename+'.log'
# don't put job name there at all if not sent
if job_name is not None:
jobstring = "#PBS -N %s" % job_name
else:
jobstring=""
if walltime is not None:
timestring = '#PBS -l walltime=' % walltime
else:
timestring = ''
if setups is not None:
if not isstring(setups):
setups = '\n'.join(setups)
else:
setups=""
script="""# vim: set ft=sh :
#PBS -S /bin/bash
#PBS -l nodes={nodes}:ppn={ppn}
#PBS -q {queue}
#PBS -j oe
#PBS -o {pbslog}
#PBS -m a
#PBS -V
#PBS -r n
#PBS -W umask=0022
{timestring}
{jobstring}
{setups}
logf="{logf}"
{command}"""
if not no_redirect:
command += ''' &> "$logf"'''
script += "\n"
script=script.format(nodes=nodes,
ppn=ppn,
queue=queue,
pbslog=pbslog,
timestring=timestring,
jobstring=jobstring,
setups=setups,
logf=logf,
command=command)
self.script=script
def __init__(self,
filename,
commands,
job_name=None,
queue='fast',
walltime=None, # e.g. '24:00:00',
nodes=1,
ppn=1,
setups=None,
buffer=False):
filename = expand_path(filename)
self.filename = filename
pbslog=filename+'.pbslog'
logf = filename+'.log'
# don't put job name there at all if not sent
if job_name is not None:
jobstring = "#PBS -N %s" % job_name
else:
jobstring=""
if walltime is not None:
timestring = '#PBS -l walltime=' % walltime
else:
timestring = ''
if setups is not None:
if not isstring(setups):
setups = '\n'.join(setups)
else:
setups=""
# now the python commands
if isinstance(commands, basestring):
commands = [commands]
commands = [c.strip() for c in commands]
commands = '\n'.join(commands)
if buffer:
ex='python'
else:
ex='python -u'
self.script="""
#PBS -S /bin/bash
#PBS -l nodes={nodes}:ppn={ppn}
#PBS -q {queue}
#PBS -j oe
#PBS -o {pbslog}
#PBS -m a
#PBS -V
#PBS -r n
#PBS -W umask=0022
{timestring}
{jobstring}
{setups}
logf="{logf}"
{ex} &> "$logf" <<EOF
{commands}
EOF
\n""".format(nodes=nodes,
ppn=ppn,
queue=queue,
pbslog=pbslog,
timestring=timestring,
jobstring=jobstring,
setups=setups,
logf=logf,
ex=ex,
commands=commands)
def create_pbs(type, procrun):
import pbs
import sdsspy
proctype='regauss'
procshort='rg'
minscore=0.1
#queue = 'slow'
#queue = 'batch'
queue = 'fast'
pbsdir=path_join('~/pbs',proctype,procrun)
pbsdir = expand_path(pbsdir)
if not os.path.exists(pbsdir):
os.makedirs(pbsdir)
espy_v = "-r ~esheldon/exports/espy-work"
esutil_v="-r ~esheldon/exports/esutil-work"
stomp_v="-r ~esheldon/exports/stomp-work"
admom_v="-r ~esheldon/exports/admom-work"
fimage_v="-r ~esheldon/exports/fimage-work"
sdsspy_v="-r ~esheldon/exports/sdsspy-work"
photo_redux = os.environ['PHOTO_REDUX']
photo_sweep = os.environ["PHOTO_SWEEP"]
photo_resolve = os.environ["PHOTO_RESOLVE"]
photo_calib = os.environ["PHOTO_CALIB"]
setups = [espy_v,esutil_v,stomp_v,admom_v,fimage_v,sdsspy_v]
setups = ['setup '+s for s in setups]
setups += ['export PHOTO_REDUX=%s' % photo_redux,
'export PHOTO_SWEEP=%s' % photo_sweep,
'export PHOTO_CALIB=%s' % photo_calib,
'export PHOTO_RESOLVE=%s' % photo_resolve]
win = sdsspy.window.Window()
runs, reruns = win.runlist(minscore)
runs.sort()
print("writing to",pbsdir)
for run in runs:
for camcol in [1,2,3,4,5,6]:
# create the pbs file name
rstr=sdsspy.files.run2string(run)
job_name = '%s%s-%s-%s' % (procshort,type,run,camcol)
pbsfilename='%s%s-%s-%s-%s.pbs' % (procshort, type, procrun, rstr, camcol)
pbsfilename=path_join(pbsdir, pbsfilename)
# the python commands to execute
python_commands = """
from lensing.regauss import RegaussSweep
from es_sdsspy import sweeps
p=sweeps.Proc('%s','%s','%s')
p.process_runs(RegaussSweep, runs=%s, camcols=%s)""" % (proctype,procrun,type,run,camcol)
ppy = pbs.PBSPython(pbsfilename,
python_commands,
job_name=job_name,
setups=setups,
queue=queue)
ppy.write()
def create_condor(type, procrun):
import pbs
import sdsspy
proctype='regauss'
procshort='rg'
minscore=0.1
condor_dir=path_join('~/condor/sweep_reduce',proctype,procrun)
condor_dir = expand_path(condor_dir)
if not os.path.exists(condor_dir):
os.makedirs(condor_dir)
win = sdsspy.window.Window()
runs, reruns = win.runlist(minscore)
runs.sort()
print("writing to",condor_dir)
for run in runs:
for camcol in [1,2,3,4,5,6]:
# create the condor file name
rstr=sdsspy.files.run2string(run)
job_name = '%s%s-%s-%s' % (procshort,type,run,camcol)
script_base='%s%s-%s-%s-%s' % (procshort, type, procrun, rstr, camcol)
condor_filename=script_base+'.condor'
script_filename=script_base+'.py'
condor_filename=path_join(condor_dir, condor_filename)
script_filename=path_join(condor_dir, script_filename)
condor_script="""
Universe = vanilla
Notification = Error
GetEnv = True
Notify_user = [email protected]
Requirements = (CPU_Experiment == "astro") && (TotalSlots == 12 || TotalSlots == 8)
+Experiment = "astro"
Initialdir = {proc_dir}
Executable = {script_base}.py
Output = {script_base}.out
Error = {script_base}.err
Log = {script_base}.log
Queue\n""".format(proc_dir=condor_dir,
script_base=script_base)
script="""#!{executable} -u
import os
os.environ['PHOTO_SWEEP'] = 'hdfs:///user/esheldon/boss/sweeps/dr8_final'
os.environ['SWEEP_REDUCE'] = 'hdfs:///user/esheldon/sweep-reduce'
from lensing.regauss import RegaussSweep
from es_sdsspy import sweeps
p=sweeps.Proc('{proctype}','{procrun}','{sweep_type}')
p.process_runs(RegaussSweep, runs={run}, camcols={camcol})
\n""".format(executable=sys.executable,
proctype=proctype,
procrun=procrun,
sweep_type=type,
run=run,
camcol=camcol)
print(condor_filename)
with open(condor_filename,'w') as fobj:
fobj.write(condor_script)
with open(script_filename,'w') as fobj:
fobj.write(script)
os.system('chmod 755 '+script_filename)