def process_image(fn, ext, nom, sfd, opt, obs, tiles):
db = opt.db
print('Reading', fn)
if sfd is None:
sfd = gSFD
# Read primary FITS header
phdr = fitsio.read_header(fn)
obstype = phdr.get('OBSTYPE','').strip()
print('obstype:', obstype)
exptime = phdr.get('EXPTIME', 0)
expnum = phdr.get('EXPNUM', 0)
filt = phdr.get('FILTER', None)
if filt is not None:
filt = filt.strip()
filt = filt.split()[0]
if filt is None:
filt = ''
airmass = phdr.get('AIRMASS', 0.)
ra = hmsstring2ra (phdr.get('RA', '0'))
dec = dmsstring2dec(phdr.get('DEC', '0'))
# Write QA plots to files named by the exposure number
print('Exposure number:', expnum)
skip = False
if obstype in ['zero', 'focus', 'dome flat', '']:
print('Skipping obstype =', obstype)
skip = True
if exptime == 0:
print('Exposure time EXPTIME in header =', exptime)
skip = True
if expnum == '':
print('No expnum in header')
skip = True
if filt == 'solid':
print('Solid (block) filter.')
skip = True
if skip and not db:
return None
if db:
import obsdb
if ext is None:
ext = get_default_extension(fn)
m,created = obsdb.MeasuredCCD.objects.get_or_create(
filename=fn, extension=ext)
m.obstype = obstype
m.camera = camera_name(phdr)
m.expnum = expnum
m.exptime = exptime
m.mjd_obs = phdr.get('MJD-OBS', 0.)
m.airmass = airmass
m.rabore = ra
m.decbore = dec
m.band = filt
m.bad_pixcnt = ('PIXCNT1' in phdr)
m.readtime = phdr.get('READTIME', 0.)
if opt.focus and obstype == 'focus' and m.camera == 'mosaic3':
from mosaic_focus import Mosaic3FocusMeas
show_plot = opt.show
if show_plot:
import pylab as plt
plt.figure(2, figsize=(8,10))
if ext is None:
ext = get_default_extension(fn)
meas = Mosaic3FocusMeas(fn, ext, nom)
focusfn = 'focus.png'
meas.run(ps=None, plotfn=focusfn)
print('Wrote', focusfn)
if show_plot:
plt.draw()
plt.show(block=False)
plt.pause(0.001)
plt.figure(1)
if skip:
m.save()
return None
if opt.doplots:
from astrometry.util.plotutils import PlotSequence
ps = PlotSequence('qa-%i' % expnum)
ps.printfn = False
else:
ps = None
# Measure the new image
kwa = {}
if ext is not None:
kwa.update(ext=ext)
if opt.n_fwhm is not None:
kwa.update(n_fwhm=opt.n_fwhm)
M = measure_raw(fn, ps=ps, **kwa)
if opt.doplots:
from glob import glob
# Gather all the QAplots into a single pdf and clean them up.
qafile = 'qa-%i.pdf' % expnum
pnglist = sorted(glob('qa-%i-??.png' % expnum))
cmd = 'convert {} {}'.format(' '.join(pnglist), qafile)
print('Writing out {}'.format(qafile))
#print(cmd)
os.system(cmd)
if not opt.keep_plots:
[os.remove(png) for png in pnglist]
# (example results for testig)
#M = {'seeing': 1.4890481099577366, 'airmass': 1.34,
#'skybright': 18.383479116033314, 'transparency': 0.94488537276869045,
#'band': 'z', 'zp': 26.442847814941093}
#print('Measurements:', M)
trans = M.get('transparency', 0)
band = M['band']
# Look up E(B-V) in SFD map
ebv = sfd.ebv(ra, dec)[0]
print('E(B-V): %.3f' % ebv)
if trans > 0:
fid = nom.fiducial_exptime(band)
expfactor = exposure_factor(fid, nom,
airmass, ebv, M['seeing'], M['skybright'],
trans)
print('Exposure factor: %6.3f' % expfactor)
t_exptime = expfactor * fid.exptime
print('Target exposure time: %6.1f' % t_exptime)
t_exptime = np.clip(t_exptime, fid.exptime_min, fid.exptime_max)
print('Clipped exposure time: %6.1f' % t_exptime)
if band == 'z':
t_sat = nom.saturation_time(band, M['skybright'])
if t_exptime > t_sat:
t_exptime = t_sat
print('Reduced exposure time to avoid z-band saturation: %.1f' % t_exptime)
print
print('Actual exposure time taken: %6.1f' % exptime)
print('Depth (exposure time) factor: %6.3f' % (exptime / t_exptime))
# If you were going to re-plan, you would run with these args:
plandict = dict(seeing=M['seeing'], transparency=trans)
# Assume the sky is as much brighter than canonical in each band... unlikely
dsky = M['skybright'] - nom.sky(M['band'])
for b in 'grz':
plandict['sb'+b] = nom.sky(b) + dsky
# Note that nightlystrategy.py takes UTC dates.
start = datenow()
# Start the strategy 5 minutes from now.
start += datetime.timedelta(0, 5*60)
d = start.date()
plandict['startdate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
t = start.time()
plandict['starttime'] = t.strftime('%H:%M:%S')
# Make an hour-long plan
end = start + datetime.timedelta(0, 3600)
d = end.date()
plandict['enddate'] = '%04i-%02i-%02i' % (d.year, d.month, d.day)
t = end.time()
plandict['endtime'] = t.strftime('%H:%M:%S')
# Set "--date" to be the UTC date at previous sunset.
# (nightlystrategy will ask for the next setting of the sun below
# -18-degrees from that date to define the sn_18). We could
# probably also get away with subtracting, like, 12 hours from
# now()...
sun = ephem.Sun()
obs.date = datenow()
# not the proper horizon, but this doesn't matter -- just need it to
# be before -18-degree twilight.
obs.horizon = 0.
sunset = obs.previous_setting(sun)
# pyephem's Date.tuple() splits a date into y,m,d,h,m,s
d = sunset.tuple()
#print('Date at sunset, UTC:', d)
year,month,day = d[:3]
plandict['date'] = '%04i-%02i-%02i' % (year, month, day)
# Decide the pass.
goodseeing = plandict['seeing'] < 1.3
photometric = plandict['transparency'] > 0.9
if goodseeing and photometric:
passnum = 1
elif goodseeing or photometric:
passnum = 2
else:
passnum = 3
plandict['pass'] = passnum
## ??
plandict['portion'] = 1.0
print('Replan command:')
print()
print('python2.7 nightlystrategy.py --seeg %(seeing).3f --seer %(seeing).3f --seez %(seeing).3f --sbg %(sbg).3f --sbr %(sbr).3f --sbz %(sbz).3f --transparency %(transparency).3f --start-date %(startdate)s --start-time %(starttime)s --end-date %(enddate)s --end-time %(endtime)s --date %(date)s --portion %(portion)f --pass %(pass)i' % plandict)
print()
else:
plandict = None
expfactor = 0.
rtn = (M, plandict, expnum)
if not db:
return rtn
m.racenter = M['ra_ccd']
m.deccenter = M['dec_ccd']
m.ebv = ebv
zp = M.get('zp', 0.)
if zp is None:
zp = 0.
m.zeropoint = zp
m.transparency = trans
m.seeing = M.get('seeing', 0.)
m.sky = M['skybright']
m.expfactor = expfactor
m.dx = M.get('dx', 0)
m.dy = M.get('dy', 0)
m.nmatched = M.get('nmatched',0)
img = fitsio.read(fn, ext=1)
cheaphash = np.sum(img)
# cheaphash becomes an int64.
m.md5sum = cheaphash
set_tile_fields(m, phdr, tiles)
m.save()
return rtn
def ptf_exposure_metadata(filenames, hdus=None, trim=None):
nan = np.nan
primkeys = [('FILTER',''),
('RA', nan),
('DEC', nan),
('AIRMASS', nan),
('DATE-OBS', ''),
('EXPTIME', nan),
('EXPNUM', 0),
('MJD-OBS', 0),
('PROPID', ''),
]
hdrkeys = [('AVSKY', nan),
('ARAWGAIN', nan),
('FWHM', nan),
('CRPIX1',nan),
('CRPIX2',nan),
('CRVAL1',nan),
('CRVAL2',nan),
('CD1_1',nan),
('CD1_2',nan),
('CD2_1',nan),
('CD2_2',nan),
('EXTNAME',''),
('CCDNAME',''),
('CCDNUM',''),
('CAMERA',''),
]
otherkeys = [('IMAGE_FILENAME',''), ('IMAGE_HDU',0),
('HEIGHT',0),('WIDTH',0),
]
allkeys = primkeys + hdrkeys + otherkeys
#for each hdu and file, append ptf header info to vals
vals = dict([(k,[]) for k,d in allkeys])
for i,fn in enumerate(filenames):
print('Reading', (i+1), 'of', len(filenames), ':', fn)
F = fitsio.FITS(fn)
cpfn = fn
if trim is not None:
cpfn = cpfn.replace(trim, '')
if hdus is not None:
hdulist = hdus
else:
hdulist = range(0, len(F))
#loop through hdus
for hdu in hdulist:
for k,d in allkeys: #d is not used below
if k in F[hdu].read_header().keys():
vals[k].append(F[hdu].read_header()[k])
else:
continue #will be set below
#special handling
H,W = F[hdu].get_info()['dims']
vals['HEIGHT'].append(H)
vals['WIDTH'].append(W)
vals['IMAGE_HDU'].append(hdu)
vals['AVSKY'].append(0.) #estimate of sky level, ok to set to 0 in legacypipe
vals['EXTNAME'].append(os.path.basename(fn)[-8:-5]) #CCD id, ex) N16 for DECam
vals['CCDNAME'][-1]= os.path.basename(fn)[-8:-5]
vals['FILTER'][-1]= vals['FILTER'][-1].strip().lower() #unique ptf band name
vals['EXPNUM'].append(0) #default value givein in function "exposure_metadata"
vals['CAMERA'].append('ptf') #default value givein in function "exposure_metadata"
vals['IMAGE_FILENAME'].append(os.path.basename(fn))
#diff naming convenctions between (DECaLS,PTF)
map=[
('RA', 'OBJRA'),
('DEC', 'OBJDEC'),
('MJD-OBS', 'OBSMJD'),
('PROPID', 'PTFPID'),
('ARAWGAIN', 'GAIN'),
('FWHM', 'MEDFWHM'),
('CCDNUM','CCDID'),
]
for decal,ptf in map:
try: vals[decal].append(F[hdu].read_header()[ptf])
except AttributeError: print "WARNING, could not find ",decal,"or",ptf
#for k,d in allkeys: print "FINAL vals: k= ",k,"vals[k]= ",vals[k]
#header info now stord in val dict
T = fits_table()
for k,d in allkeys:
T.set(k.lower().replace('-','_'), np.array(vals[k]))
#finish naming conventions
T.filter = np.array([s.split()[0] for s in T.filter])
#KJB LEFT OFF HERE
T.ra_bore = np.array([hmsstring2ra (s) for s in T.ra ])
T.dec_bore = np.array([dmsstring2dec(s) for s in T.dec])
T.ra = np.zeros(len(T))
T.dec = np.zeros(len(T))
for i in range(len(T)):
W,H = T.width[i], T.height[i]
wcs = Tan(T.crval1[i], T.crval2[i], T.crpix1[i], T.crpix2[i],
T.cd1_1[i], T.cd1_2[i], T.cd2_1[i], T.cd2_2[i], float(W), float(H))
xc,yc = W/2.+0.5, H/2.+0.5
rc,dc = wcs.pixelxy2radec(xc,yc)
T.ra [i] = rc
T.dec[i] = dc
#anything with type int64 crashes fits.write()
T.expnum= T.expnum.astype(np.int16)
T.image_hdu= T.image_hdu.astype(np.int16)
T.height= T.height.astype(np.int16)
T.width= T.width.astype(np.int16)
#for c in T.columns():
#if type(T.get(c)[0]) is np.int64: T.c= T.c.astype(np.int16) #answer='yes'
#else:answer='no'
#print('column= ',c,"type= ",type(T.get(c)),"type is int64?",answer)
#sanity check
for c in T.columns(): print (c,T.get(c))
return T
def exposure_metadata(filenames, hdus=None, trim=None):
'''
Creates a CCD table row object by reading metadata from a FITS
file header.
Parameters
----------
filenames : list of strings
Filenames to read
hdus : list of integers; None to read all HDUs
List of FITS extensions (HDUs) to read
trim : string
String to trim off the start of the *filenames* for the
*image_filename* table entry
Returns
-------
A table that looks like the CCDs table.
'''
nan = np.nan
primkeys = [('FILTER',''),
('RA', nan),
('DEC', nan),
('AIRMASS', nan),
('DATE-OBS', ''),
('EXPTIME', nan),
('EXPNUM', 0),
('MJD-OBS', 0),
('PROPID', ''),
('INSTRUME', ''),
('SEEING', nan),
]
hdrkeys = [('AVSKY', nan),
('ARAWGAIN', nan),
('FWHM', nan),
('CRPIX1',nan),
('CRPIX2',nan),
('CRVAL1',nan),
('CRVAL2',nan),
('CD1_1',nan),
('CD1_2',nan),
('CD2_1',nan),
('CD2_2',nan),
('EXTNAME',''),
('CCDNUM',''),
]
otherkeys = [('IMAGE_FILENAME',''), ('IMAGE_HDU',0),
('HEIGHT',0),('WIDTH',0),
]
allkeys = primkeys + hdrkeys + otherkeys
vals = dict([(k,[]) for k,d in allkeys])
for i,fn in enumerate(filenames):
print('Reading', (i+1), 'of', len(filenames), ':', fn)
F = fitsio.FITS(fn)
primhdr = F[0].read_header()
expstr = '%08i' % primhdr.get('EXPNUM')
cpfn = fn
if trim is not None:
cpfn = cpfn.replace(trim, '')
print('CP fn', cpfn)
if hdus is not None:
hdulist = hdus
else:
hdulist = range(1, len(F))
for hdu in hdulist:
hdr = F[hdu].read_header()
info = F[hdu].get_info()
#'extname': 'S1', 'dims': [4146L, 2160L]
H,W = info['dims']
for k,d in primkeys:
vals[k].append(primhdr.get(k, d))
for k,d in hdrkeys:
vals[k].append(hdr.get(k, d))
vals['IMAGE_FILENAME'].append(cpfn)
vals['IMAGE_HDU'].append(hdu)
vals['WIDTH'].append(int(W))
vals['HEIGHT'].append(int(H))
T = fits_table()
for k,d in allkeys:
T.set(k.lower().replace('-','_'), np.array(vals[k]))
#T.about()
# DECam: INSTRUME = 'DECam'
T.rename('instrume', 'camera')
T.camera = np.array([t.lower().strip() for t in T.camera])
#T.rename('extname', 'ccdname')
T.ccdname = np.array([t.strip() for t in T.extname])
T.filter = np.array([s.split()[0] for s in T.filter])
T.ra_bore = np.array([hmsstring2ra (s) for s in T.ra ])
T.dec_bore = np.array([dmsstring2dec(s) for s in T.dec])
T.ra = np.zeros(len(T))
T.dec = np.zeros(len(T))
for i in range(len(T)):
W,H = T.width[i], T.height[i]
wcs = Tan(T.crval1[i], T.crval2[i], T.crpix1[i], T.crpix2[i],
T.cd1_1[i], T.cd1_2[i], T.cd2_1[i], T.cd2_2[i], float(W), float(H))
xc,yc = W/2.+0.5, H/2.+0.5
rc,dc = wcs.pixelxy2radec(xc,yc)
T.ra [i] = rc
T.dec[i] = dc
return T
def exposure_metadata(filenames, hdus=None, trim=None):
'''
Creates a CCD table row object by reading metadata from a FITS
file header.
Parameters
----------
filenames : list of strings
Filenames to read
hdus : list of integers; None to read all HDUs
List of FITS extensions (HDUs) to read
trim : string
String to trim off the start of the *filenames* for the
*image_filename* table entry
Returns
-------
A table that looks like the CCDs table.
'''
nan = np.nan
primkeys = [('FILTER',''),
('RA', nan),
('DEC', nan),
('AIRMASS', nan),
('DATE-OBS', ''),
('EXPTIME', nan),
('EXPNUM', 0),
('MJD-OBS', 0),
('PROPID', ''),
('INSTRUME', ''),
]
hdrkeys = [('AVSKY', nan),
('ARAWGAIN', nan),
('FWHM', nan),
('CRPIX1',nan),
('CRPIX2',nan),
('CRVAL1',nan),
('CRVAL2',nan),
('CD1_1',nan),
('CD1_2',nan),
('CD2_1',nan),
('CD2_2',nan),
('EXTNAME',''),
('CCDNUM',''),
]
otherkeys = [('IMAGE_FILENAME',''), ('IMAGE_HDU',0),
('HEIGHT',0),('WIDTH',0),
]
allkeys = primkeys + hdrkeys + otherkeys
vals = dict([(k,[]) for k,d in allkeys])
for i,fn in enumerate(filenames):
print('Reading', (i+1), 'of', len(filenames), ':', fn)
F = fitsio.FITS(fn)
primhdr = F[0].read_header()
expstr = '%08i' % primhdr.get('NOCID') #'EXPNUM')
# # Parse date with format: 2014-08-09T04:20:50.812543
# date = datetime.datetime.strptime(primhdr.get('DATE-OBS'),
# '%Y-%m-%dT%H:%M:%S.%f')
# # Subract 12 hours to get the date used by the CP to label the night;
# # CP20140818 includes observations with date 2014-08-18 evening and
# # 2014-08-19 early AM.
# cpdate = date - datetime.timedelta(0.5)
# #cpdatestr = '%04i%02i%02i' % (cpdate.year, cpdate.month, cpdate.day)
# #print 'Date', date, '-> CP', cpdatestr
# cpdateval = cpdate.year * 10000 + cpdate.month * 100 + cpdate.day
# print 'Date', date, '-> CP', cpdateval
cpfn = fn
if trim is not None:
cpfn = cpfn.replace(trim, '')
print('CP fn', cpfn)
if hdus is not None:
hdulist = hdus
else:
hdulist = range(1, len(F))
for hdu in hdulist:
hdr = F[hdu].read_header()
info = F[hdu].get_info()
#'extname': 'S1', 'dims': [4146L, 2160L]
H,W = info['dims']
for k,d in primkeys:
vals[k].append(primhdr.get(k, d))
for k,d in hdrkeys:
vals[k].append(hdr.get(k, d))
vals['IMAGE_FILENAME'].append( os.path.join('mosaic/',os.path.basename(cpfn)) )
vals['ARAWGAIN'][-1]= hdr.get('GAIN') #e/ADU
vals['FWHM'][-1]= hdr.get('SEEING1')/0.258 #pixscale=0.258 arcsec/pix
vals['IMAGE_HDU'].append(hdu)
vals['WIDTH'].append(int(W))
vals['HEIGHT'].append(int(H))
T = fits_table()
for k,d in allkeys:
T.set(k.lower().replace('-','_'), np.array(vals[k]))
#T.about()
# DECam: INSTRUME = 'DECam'
T.rename('instrume', 'camera')
T.camera = np.array(['mosaic' for t in T.camera]) #t.lower() for t in T.camera])
#T.rename('extname', 'ccdname')
T.ccdname = np.array([t.strip() for t in T.extname])
T.filter = np.array(['z' for s in T.filter]) #s.split()[0] for s in T.filter])
T.ra_bore = np.array([hmsstring2ra (s) for s in T.ra ])
T.dec_bore = np.array([dmsstring2dec(s) for s in T.dec])
T.ra = np.zeros(len(T))
T.dec = np.zeros(len(T))
for i in range(len(T)):
W,H = T.width[i], T.height[i]
wcs = Tan(T.crval1[i], T.crval2[i], T.crpix1[i], T.crpix2[i],
T.cd1_1[i], T.cd1_2[i], T.cd2_1[i], T.cd2_2[i], float(W), float(H))
xc,yc = W/2.+0.5, H/2.+0.5
rc,dc = wcs.pixelxy2radec(xc,yc)
T.ra [i] = rc
T.dec[i] = dc
#
# T.ra = np.zeros(len(T))
# T.dec = np.zeros(len(T))
# for i in range(len(T)):
# W,H = T.width[i], T.height[i]
#
# wcs = Tan(T.crval1[i], T.crval2[i], T.crpix1[i], T.crpix2[i],
# T.cd1_1[i], T.cd1_2[i], T.cd2_1[i], T.cd2_2[i], float(W), float(H))
#
# xc,yc = W/2.+0.5, H/2.+0.5
# rc,dc = wcs.pixelxy2radec(xc,yc)
# T.ra [i] = rc
# T.dec[i] = dc
return T
def exposure_metadata(filenames, hdus=None, trim=None):
'''
Creates a CCD table row object by reading metadata from a FITS
file header.
Parameters
----------
filenames : list of strings
Filenames to read
hdus : list of integers; None to read all HDUs
List of FITS extensions (HDUs) to read
trim : string
String to trim off the start of the *filenames* for the
*image_filename* table entry
Returns
-------
A table that looks like the CCDs table.
'''
nan = np.nan
primkeys = [('FILTER',''),
('RA', nan),
('DEC', nan),
('AIRMASS', nan),
('DATE-OBS', ''),
('EXPTIME', nan),
('EXPNUM', 0),
('JULIAN', 0),
('PROPID', ''),
('INSTRUME', ''),
('SEEING', nan),
]
hdrkeys = [('SKYVAL', nan),
('GAIN', nan),
('FWHM', nan),
('CRPIX1',nan),
('CRPIX2',nan),
('CRVAL1',nan),
('CRVAL2',nan),
('CD1_1',nan),
('CD1_2',nan),
('CD2_1',nan),
('CD2_2',nan),
('CCDNAME',''),
('CCDNUM',''),
]
otherkeys = [('IMAGE_FILENAME',''), ('IMAGE_HDU',0),
('HEIGHT',0),('WIDTH',0),
]
allkeys = primkeys + hdrkeys + otherkeys
vals = dict([(k,[]) for k,d in allkeys])
for i,fn in enumerate(filenames):
print('Reading', (i+1), 'of', len(filenames), ':', fn)
F = fits.open(fn)
primhdr = F[0].header
expstr = '%08i' % primhdr['IMAGEID'] #EXPNUM']
# # Parse date with format: 2014-08-09T04:20:50.812543
# date = datetime.datetime.strptime(primhdr.get('DATE-OBS'),
# '%Y-%m-%dT%H:%M:%S.%f')
# # Subract 12 hours to get the date used by the CP to label the night;
# # CP20140818 includes observations with date 2014-08-18 evening and
# # 2014-08-19 early AM.
# cpdate = date - datetime.timedelta(0.5)
# #cpdatestr = '%04i%02i%02i' % (cpdate.year, cpdate.month, cpdate.day)
# #print 'Date', date, '-> CP', cpdatestr
# cpdateval = cpdate.year * 10000 + cpdate.month * 100 + cpdate.day
# print 'Date', date, '-> CP', cpdateval
cpfn = fn
if trim is not None:
cpfn = cpfn.replace(trim, '')
print('CP fn', cpfn)
if hdus is not None:
hdulist = hdus
else:
hdulist = range(1, len(F))
for hdu in hdulist:
hdr = F[hdu].header
#'extname': 'S1', 'dims': [4146L, 2160L]
W,H = hdr['NAXIS1'],hdr['NAXIS2']
for k,d in primkeys:
vals[k].append(primhdr.get(k, d))
for k,d in hdrkeys:
vals[k].append(hdr.get(k, d))
vals['IMAGE_FILENAME'].append( '90prime/'+os.path.basename(cpfn) )
vals['IMAGE_HDU'].append(hdu)
vals['WIDTH'].append(int(W))
vals['HEIGHT'].append(int(H))
#WARNING, is this Bok exposure number?
vals['EXPNUM'][-1]= int(F[0].header['IMAGEID'])
T = fits_table()
for k,d in allkeys:
T.set(k.lower().replace('-','_'), np.array(vals[k]))
#T.about()
# DECam: INSTRUME = 'DECam'
T.rename('INSTRUME'.lower(), 'camera')
T.camera = np.array([t.lower() for t in T.camera])
T.rename('gain', 'ARAWGAIN'.lower()) #ARAWGAIN is name for gain from decam parser
T.rename('SKYVAL'.lower(), 'AVSKY'.lower()) #AVSKY is name for sky from decam parser
T.rename('JULIAN'.lower(), 'MJD_OBS'.lower()) #MJD-OBS is name for julian from decam parser
T.ccdname = np.array([t.strip() for t in T.ccdname])
T.extname = np.array([t.strip() for t in T.ccdname])
T.ccdnum = np.array([t.strip()[-1] for t in T.ccdname])
T.filter = np.array([s.strip()[0] for s in T.filter])
T.filter[T.filter == 'b']= 'r' #bok g is 'g' but bok r is 'bokr' so this became b with above
T.ra_bore = np.array([hmsstring2ra (s) for s in T.ra ])
T.dec_bore = np.array([dmsstring2dec(s) for s in T.dec])
T.ra = np.zeros(len(T))
T.dec = np.zeros(len(T))
for i in range(len(T)):
W,H = T.width[i], T.height[i]
wcs = Tan(T.crval1[i], T.crval2[i], T.crpix1[i], T.crpix2[i],
T.cd1_1[i], T.cd1_2[i], T.cd2_1[i], T.cd2_2[i], float(W), float(H))
xc,yc = W/2.+0.5, H/2.+0.5
rc,dc = wcs.pixelxy2radec(xc,yc)
T.ra [i] = rc
T.dec[i] = dc
return T
def ptf_exposure_metadata(filenames, hdus=None, trim=None):
nan = np.nan
primkeys = [
('FILTER', ''),
('RA', nan),
('DEC', nan),
('AIRMASS', nan),
('DATE-OBS', ''),
('EXPTIME', nan),
('EXPNUM', 0),
('MJD-OBS', 0),
('PROPID', ''),
]
hdrkeys = [
('AVSKY', nan),
('ARAWGAIN', nan),
('FWHM', nan),
('CRPIX1', nan),
('CRPIX2', nan),
('CRVAL1', nan),
('CRVAL2', nan),
('CD1_1', nan),
('CD1_2', nan),
('CD2_1', nan),
('CD2_2', nan),
('EXTNAME', ''),
('CCDNAME', ''),
('CCDNUM', ''),
('CAMERA', ''),
]
otherkeys = [
('IMAGE_FILENAME', ''),
('IMAGE_HDU', 0),
('HEIGHT', 0),
('WIDTH', 0),
]
allkeys = primkeys + hdrkeys + otherkeys
#for each hdu and file, append ptf header info to vals
vals = dict([(k, []) for k, d in allkeys])
for i, fn in enumerate(filenames):
print('Reading', (i + 1), 'of', len(filenames), ':', fn)
F = fitsio.FITS(fn)
cpfn = fn
if trim is not None:
cpfn = cpfn.replace(trim, '')
if hdus is not None:
hdulist = hdus
else:
hdulist = range(0, len(F))
#loop through hdus
for hdu in hdulist:
for k, d in allkeys: #d is not used below
if k in F[hdu].read_header().keys():
vals[k].append(F[hdu].read_header()[k])
else:
continue #will be set below
#special handling
H, W = F[hdu].get_info()['dims']
vals['HEIGHT'].append(H)
vals['WIDTH'].append(W)
vals['IMAGE_HDU'].append(hdu)
vals['AVSKY'].append(
0.) #estimate of sky level, ok to set to 0 in legacypipe
vals['EXTNAME'].append(
os.path.basename(fn)[-8:-5]) #CCD id, ex) N16 for DECam
vals['CCDNAME'][-1] = os.path.basename(fn)[-8:-5]
vals['FILTER'][-1] = vals['FILTER'][-1].strip().lower(
) #unique ptf band name
vals['EXPNUM'].append(
0) #default value givein in function "exposure_metadata"
vals['CAMERA'].append(
'ptf') #default value givein in function "exposure_metadata"
vals['IMAGE_FILENAME'].append(os.path.basename(fn))
#diff naming convenctions between (DECaLS,PTF)
map = [
('RA', 'OBJRA'),
('DEC', 'OBJDEC'),
('MJD-OBS', 'OBSMJD'),
('PROPID', 'PTFPID'),
('ARAWGAIN', 'GAIN'),
('FWHM', 'MEDFWHM'),
('CCDNUM', 'CCDID'),
]
for decal, ptf in map:
try:
vals[decal].append(F[hdu].read_header()[ptf])
except AttributeError:
print "WARNING, could not find ", decal, "or", ptf
#for k,d in allkeys: print "FINAL vals: k= ",k,"vals[k]= ",vals[k]
#header info now stord in val dict
T = fits_table()
for k, d in allkeys:
T.set(k.lower().replace('-', '_'), np.array(vals[k]))
#finish naming conventions
T.filter = np.array([s.split()[0] for s in T.filter])
#KJB LEFT OFF HERE
T.ra_bore = np.array([hmsstring2ra(s) for s in T.ra])
T.dec_bore = np.array([dmsstring2dec(s) for s in T.dec])
T.ra = np.zeros(len(T))
T.dec = np.zeros(len(T))
for i in range(len(T)):
W, H = T.width[i], T.height[i]
wcs = Tan(T.crval1[i], T.crval2[i], T.crpix1[i], T.crpix2[i],
T.cd1_1[i], T.cd1_2[i], T.cd2_1[i], T.cd2_2[i], float(W),
float(H))
xc, yc = W / 2. + 0.5, H / 2. + 0.5
rc, dc = wcs.pixelxy2radec(xc, yc)
T.ra[i] = rc
T.dec[i] = dc
#anything with type int64 crashes fits.write()
T.expnum = T.expnum.astype(np.int16)
T.image_hdu = T.image_hdu.astype(np.int16)
T.height = T.height.astype(np.int16)
T.width = T.width.astype(np.int16)
#for c in T.columns():
#if type(T.get(c)[0]) is np.int64: T.c= T.c.astype(np.int16) #answer='yes'
#else:answer='no'
#print('column= ',c,"type= ",type(T.get(c)),"type is int64?",answer)
#sanity check
for c in T.columns():
print(c, T.get(c))
return T
'wget -P /tmp https://raw.githubusercontent.com/mattiaverga/OpenNGC/master/NGC.csv'
)
names = ('name', 'type', 'ra_hms', 'dec_dms', 'const', 'majax', 'minax', 'pa',
'bmag', 'vmag', 'jmag', 'hmag', 'kmag', 'sbrightn', 'hubble',
'cstarumag', 'cstarbmag', 'cstarvmag', 'messier', 'ngc', 'ic',
'cstarnames', 'identifiers', 'commonnames', 'nednotes', 'ongcnotes')
NGC = ascii.read('/tmp/NGC.csv', delimiter=';', names=names)
NGC = NGC[(NGC['ra_hms'] != 'N/A')]
ra, dec = [], []
for _ra, _dec in zip(ma.getdata(NGC['ra_hms']), ma.getdata(NGC['dec_dms'])):
ra.append(
hmsstring2ra(_ra.replace('h', ':').replace('m', ':').replace('s', '')))
dec.append(
dmsstring2dec(
_dec.replace('d', ':').replace('m', ':').replace('s', '')))
NGC['ra'] = ra
NGC['dec'] = dec
objtype = np.char.strip(ma.getdata(NGC['type']))
# Keep all globular clusters and planetary nebulae
keeptype = ('PN', 'GCl')
keep = np.zeros(len(NGC), dtype=bool)
for otype in keeptype:
ww = [otype == tt for tt in objtype]
keep = np.logical_or(keep, ww)
print(np.sum(keep))
clusters = NGC[keep]