forked from dstndstn/tractor
/
runslice.py
executable file
·273 lines (211 loc) · 7.59 KB
/
runslice.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
#! /usr/bin/env python
'''
This is a PBS driver script we used on riemann to perform
WISE-from-SDSS forced photometry.
Jobs are submitted like:
qsub -l "nodes=1:ppn=1" -l "walltime=3:00:00" -N w1v4 -o w1v4.log -q batch -t 100-190 ./runslice.py
The "-t" option (job arrays) causes the PBS_ARRAYID environment
variable to be set for each job. We use that to determine which chunk
of work that job is going to do.
In the case of the eBOSS W3 test, we split the region into boxes in
RA,Dec and used the job id for the Dec slice (90 slices).
(I was also using this as a non-PBS command-line driver; batch=False
then)
'''
import os
import sys
from astrometry.util.ttime import *
import logging
if __name__ == '__main__':
Time.add_measurement(MemMeas)
Time.add_measurement(IoMeas)
lvl = logging.INFO
logging.basicConfig(level=lvl, format='%(message)s', stream=sys.stdout)
import numpy as np
from astrometry.util.multiproc import multiproc
from astrometry.util.file import *
# duck-type command-line options
class myopts(object):
pass
def main():
batch = False
arr = os.environ.get('PBS_ARRAYID')
if arr is not None:
arr = int(arr)
batch = True
# This gets set when running runslice.py from the command-line within an interactive job...
d = os.environ.get('PBS_O_WORKDIR')
if batch and d is not None:
os.chdir(d)
sys.path.append(os.getcwd())
opt = myopts()
opt.nonsdss = False
ps = None
if True:
# W3 area
if arr is None:
# which slice to do for interactive jobs
#arr = 147
#arr = 148
arr = 149
opt.sources = 'objs-eboss-w3-dr9.fits'
NDEC = 50
NRA = 90
band = int(arr / 100)
ri = arr % 100
print 'Band', band
print 'RA slice', ri
r0,r1 = 210.593, 219.132
d0,d1 = 51.1822, 54.1822
basedir = '/clusterfs/riemann/raid000/bosswork/boss/wise_frames'
opt.wisedatadirs = [(basedir, 'merged'),]
# basedir = '/clusterfs/riemann/raid000/bosswork/boss/wise1test'
# opt.wisedatadirs = [(os.path.join(basedir, 'allsky'), 'cryo'),
# (os.path.join(basedir, 'prelim_postcryo'), 'post-cryo'),]
opt.minflux = None
opt.wsources = 'wise-objs-w3.fits'
opt.bandnum = band
opt.osources = None
opt.minsb = 0.005
opt.ptsrc = False
opt.pixpsf = False
if False:
# eboss w3 v4
basename = 'ebossw3-v4'
opt.ptsrc = False
opt.pixpsf = False
if False:
# eboss w3 v5
basename = 'ebossw3-v5'
opt.ptsrc = True
if True:
# eboss w3 v6 (after the fact)
basename = 'eboss-w3-v6'
if not batch:
basename = 'eboss-w3-tst'
opt.ptsrc = False
opt.pixpsf = False
if False:
# Stripe82 QSO truth-table region
base = '/clusterfs/riemann/raid000/bosswork/boss/wise1ext/sdss_stripe82'
opt.sources = os.path.join(base, 'objs-eboss-stripe82-dr9.fits')
r0, r1 = 317.0, 330.0
d0, d1 = 0., 1.25
NRA = 260
NDEC = 25
if arr is None:
# which slice to do for interactive jobs
arr = 1000
band = int(arr / 1000)
ri = arr % 1000
print 'Band', band
print 'RA slice', ri
basedir = '/clusterfs/riemann/raid000/bosswork/boss/wise1test_stripe82/old'
opt.wisedatadirs = [(os.path.join(basedir, 'allsky'), 'cryo'),
(os.path.join(basedir, 'prelim_postcryo'), 'post-cryo'),]
opt.minflux = None
opt.bandnum = band
opt.osources = None
opt.minsb = 0.005
opt.ptsrc = False
opt.pixpsf = False
# v1
basename = 'eboss-s82-v1'
opt.ptsrc = False
opt.pixpsf = False
if not batch:
basename = 'eboss-s82-tst'
opt.ptsrc = False
opt.pixpsf = False
if not batch:
import matplotlib
matplotlib.use('Agg')
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence(basename)
# Has to happen *after* matplotlib init...
import wise3
dd = np.linspace(d0, d1, NDEC + 1)
rr = np.linspace(r0, r1, NRA + 1)
mp = multiproc()
tr0 = Time()
rlo,rhi = rr[ri], rr[ri+1]
for di,(dlo,dhi) in enumerate(zip(dd[:-1], dd[1:])):
fn = '%s-r%02i-d%02i-w%i.fits' % (basename, ri, di, opt.bandnum)
if os.path.exists(fn):
print 'Output file exists:', fn
print 'Skipping'
if batch:
continue
# HACK!!
#if not batch and di != 25:
# continue
td0 = Time()
try:
P = dict(ralo=rlo, rahi=rhi, declo=dlo, dechi=dhi,
opt=opt, mp=mp, ps=ps)
t00 = Time()
for stage in [ 100, 101, 102, 103, 104, 105, 106, 700 ]:
f = eval('wise3.stage%i' % stage)
ts0 = Time()
R = f(**P)
dt = Time() - ts0
P.update(R)
P.update(**{ 't%i' % stage: dt })
print 'stage', stage, ':', dt
dt = Time() - t00
P.update(**{ 'ttotal': dt })
# Grab result FITS structure
R = P['R']
R.writeto(fn)
print 'Wrote', fn
imst = P['imstats']
fn = '%s-r%02i-d%02i-w%i-imstats.fits' % (basename, ri, di, opt.bandnum)
imst.writeto(fn)
print 'Wrote', fn
pfn = '%s-r%02i-d%02i-w%i.pickle' % (basename, ri, di, opt.bandnum)
# tractor = P['tractor']
# ims1 = P['ims1']
# res1 = []
# for tim,(img,mod,ie,chi,roi) in zip(tractor.images, ims1):
# for k in ['origInvvar', 'starMask', 'inverr', 'cinvvar', 'goodmask',
# 'maskplane', 'rdmask', 'mask', 'uncplane', 'vinvvar']:
#
# ### DEBUG
# #continue
# # Debug
# if k == 'rdmask':
# continue
#
# try:
# delattr(tim, k)
# except:
# pass
#
# res1.append((tim, mod, roi))
#
# PP = dict(res1=res1, cat=tractor.getCatalog(), rd=P['rd'], ri=ri, di=di,
# bandnum=opt.bandnum, S=P['S'],
# ralo=rlo, rahi=rhi, declo=dlo, dechi=dhi,
# opt=opt,
# T=P['T'])
pickle_to_file(P, pfn)
print
print 'Dec slice time:', Time() - td0
print
except:
import traceback
print '---------------------------------------------------'
print 'FAILED: dec slice', di, 'ra slice', ri
print rlo,rhi, dlo,dhi
print '---------------------------------------------------'
traceback.print_exc()
print '---------------------------------------------------'
if not batch:
raise
print
print 'RA slice time:', Time() - tr0
print
if __name__ == '__main__':
import cProfile
from datetime import datetime
cProfile.run('main()', 'prof-runslice-%s.dat' % (datetime.now().isoformat()))