def _get_wcs(self, file_id):
"""
either load the wcs from the image_info, or from
the image header
"""
if self.psf_data is not None:
psf = self.psf_data[file_id]
if hasattr(psf, 'get_wcs'):
wcs = psf.get_wcs()
impath = self.image_info['image_path'][file_id].strip()
ext = self.image_info['image_ext'][file_id]
hdr = fitsio.read_header(impath, ext=ext)
if 'znaxis1' in hdr:
naxis = numpy.array([hdr['znaxis1'], hdr['znaxis2']])
else:
naxis = numpy.array([hdr['naxis1'], hdr['naxis2']])
wcs.set_naxis(naxis)
return wcs
if 'wcs' in self.image_info.dtype.names:
wcs_string = self.image_info['wcs'][file_id]
wcs_data = json.loads(wcs_string)
else:
impath=self.image_info['image_path'][file_id].strip()
ext=self.image_info['image_ext'][file_id]
wcs_data = fitsio.read_header(impath, ext=ext)
wcs = eu.wcsutil.WCS(wcs_data)
return wcs
def init_data_map(datadir, outdir, expTimes=None, files=None):
dataMap = {}
if not os.path.exists(outdir):
os.mkdir(outdir)
dataMap['outdir'] = outdir
if files is None:
dataMap['files'] = sorted(
glob.glob(datadir + '*.fits') + glob.glob(datadir + '*.fits.gz') +
glob.glob(datadir + '*.fits.fz'))
else:
dataMap['files'] = files
dataMap['rawFiles'] = dataMap['files']
dataMap['oscan'] = bokio.FileNameMap(outdir)
dataMap['proc'] = bokio.FileNameMap(outdir, '_p')
dataMap['files'] = [dataMap['oscan'](f) for f in dataMap['files']]
if expTimes is None:
dataMap['expTime'] = np.array(
[fitsio.read_header(f)['EXPTIME'] for f in dataMap['files']])
else:
dataMap['expTime'] = expTimes
try:
# assume they are all the same
dataMap['dataSec'] = \
_convertfitsreg(fitsio.read_header(
dataMap['files'][0],'IM4')['DATASEC'])
except IOError:
pass
return dataMap
def add_depth_tag(decals, brick, outdir, overwrite=False):
outfn = os.path.join(outdir, 'tractor', brick[:3], 'tractor-%s.fits' % brick)
if os.path.exists(outfn) and not overwrite:
print 'Exists:', outfn
return
fn = decals.find_file('tractor', brick=brick)
if not os.path.exists(fn):
print 'Does not exist:', fn
return
T = fits_table(fn, lower=False)
primhdr = fitsio.read_header(fn)
hdr = fitsio.read_header(fn, ext=1)
print 'Read', len(T), 'from', fn
T.decam_depth = np.zeros((len(T), len(decals.allbands)), np.float32)
T.decam_galdepth = np.zeros((len(T), len(decals.allbands)), np.float32)
bands = 'grz'
ibands = [decals.index_of_band(b) for b in bands]
ix = np.clip(np.round(T.bx).astype(int), 0, 3599)
iy = np.clip(np.round(T.by).astype(int), 0, 3599)
for iband,band in zip(ibands, bands):
fn = decals.find_file('depth', brick=brick, band=band)
if os.path.exists(fn):
print 'Reading', fn
img = fitsio.read(fn)
T.decam_depth[:,iband] = img[iy, ix]
fn = decals.find_file('galdepth', brick=brick, band=band)
if os.path.exists(fn):
print 'Reading', fn
img = fitsio.read(fn)
T.decam_galdepth[:,iband] = img[iy, ix]
outfn = os.path.join(outdir, 'tractor', brick[:3], 'tractor-%s.fits' % brick)
trymakedirs(outfn, dir=True)
for s in [
'Data product of the DECam Legacy Survey (DECaLS)',
'Full documentation at http://legacysurvey.org',
]:
primhdr.add_record(dict(name='COMMENT', value=s, comment=s))
# print 'Header:', hdr
# T.writeto(outfn, header=hdr, primheader=primhdr)
# Yuck, all this to get the units right
tmpfn = outfn + '.tmp'
fits = fitsio.FITS(tmpfn, 'rw', clobber=True)
fits.write(None, header=primhdr)
cols = T.get_columns()
units = []
for i in range(1, len(cols)+1):
u = hdr.get('TUNIT%i' % i, '')
units.append(u)
# decam_depth units
fluxiv = '1/nanomaggy^2'
units[-2] = fluxiv
units[-1] = fluxiv
fits.write([T.get(c) for c in cols], names=cols, header=hdr, units=units)
fits.close()
os.rename(tmpfn, outfn)
print 'Wrote', outfn
def get_moon_angle(file_name, number_of_files=100):
df_SKY=pd.read_csv(file_name)
new_SKY = df_SKY.query('OBJTYPE=="SKY".ljust(16)')
DEC = np.array(new_SKY['DEC'].tolist())
FIBERID = np.array(new_SKY['FIBERID'].tolist())
MJD = np.array(new_SKY['MJD'].tolist())
RA = np.array(new_SKY['RA'].tolist())
plates = list(new_SKY['PLATE'][:number_of_files].values)
mjds = list(new_SKY['MJD'][:number_of_files].values)
fiberids = list(new_SKY['FIBERID'][:number_of_files].values)
files = ['spec-%s-%s-%s.fits'%(plate,mjd,str(fiberid).zfill(4)) for plate,mjd,fiberid in zip(plates,mjds,fiberids)]
#FINDING INTERPLATE SKY TIME
fin_mean=[] #time for each moon sky observation interplate, used for moon angle
for r in range(len(files)): #mechanization of ubiquitious exposure count
sampling=fitsio.read_header(files[r],0) #finding number of exposures
h_beg=[]
h_end=[]
mean_per=[] #time for each moon sky observation intraplate
for k in range(4, sampling['NEXP']+4): #4 is constant for everything
h=fitsio.read_header(files[3],k)
h_beg.append(h['TAI-BEG']) #ONLY VARIES WITH PLATE NUMBER, different for k's
h_end.append(h['TAI-END']) #ONLY VARIES WITH PLATE NUMBER, different for k's
mean_times=(h['TAI-BEG']+h['TAI-END'])/2
mean_per.append(mean_times)
tot=np.mean(mean_per)
fin_mean.append(tot)
#TAI TO MJD
new_time=np.array(fin_mean)
time_MJD=new_time/(86400)
moon_coords1=[]
moon_coords=[]
for i in range (len(time_MJD)):
t=Time(time_MJD[i], format='mjd')
moon_coords1.append(get_moon(t))
moon_coords.append(moon_coords1[i].spherical)
moon_coords=list(moon_coords)
#print(moon_coords)
moon_coords2=[]
for i in range(len(moon_coords)):
moon_coords2.append(moon_coords[i]._values)
moon_coords2=np.array(moon_coords2)
sky_coords=np.vstack((RA,DEC)).T
#SPLITTING ARRAY
new_sky_coords=sky_coords[:number_of_files]
new_sky_RA = [item[0] for item in new_sky_coords]
new_sky_DEC = [item[1] for item in new_sky_coords]
moon_RA = [item[0] for item in moon_coords2]
moon_DEC = [item[1] for item in moon_coords2]
new_sky_RA=np.array(new_sky_RA)*(np.pi/180)
new_sky_DEC=np.array(new_sky_DEC)*(np.pi/180)
moon_RA=np.array(moon_RA)*(np.pi/180)
moon_DEC=np.array(moon_DEC)*(np.pi/180)
#MOON-SKY ANGLE CALCULATIONS
moon_sky_angle=[]
for i in range(len(new_sky_coords)):
moon_sky_angle.append(np.arccos((np.sin(new_sky_RA[i])*np.sin(moon_RA[i]))+np.cos(new_sky_RA[i])*np.cos(moon_RA[i])*(np.cos(new_sky_DEC[i]-moon_DEC[i])))*(180/np.pi))
#list(moon_sky_angle)
return (np.array(moon_sky_angle))
def create_output_sdfits(self, feed, window, pol, suffix=''):
try:
signalRows = self.row_list.get(self.cl.mapscans[0], feed, window, pol)
ext = signalRows['EXTENSION']
rows = signalRows['ROW']
columns = tuple(self.infile[ext].get_colnames())
firstIntegration = Integration(self.infile[ext][columns][rows[0]])
if isinstance(firstIntegration['OBJECT'],str):
targetname = firstIntegration['OBJECT'].replace(" ","")
else:
targetname = str(firstIntegration['OBJECT'],encoding='UTF-8').replace(" ", "")
except KeyError:
print('WARNING: Can not find data for scan {scan} window {win} feed {feed} polarization {pol}'.format(scan=self.cl.mapscans[0], win=window, feed=feed, pol=pol))
raise
self.outfilename = targetname +\
'_scan_' + str(self.cl.mapscans[0]) + '_' +\
str(self.cl.mapscans[-1]) + '_window' + str(window) +\
'_feed' + str(feed) + '_pol' + str(pol) + suffix +'.fits'
if self.log is None:
self.log = Logging(self.cl, self.outfilename.rstrip('.fits'))
if self.CLOBBER is False and os.path.exists(self.outdir + self.outfilename):
self.log.doMessage('WARN', ' Will not overwrite existing pipeline output.\nConsider using \'--clobber\' option to overwrite.')
sys.exit()
# create a new table
old_stdout = sys.stdout
try:
from cStringIO import StringIO
except ImportError:
from io import StringIO
sys.stdout = StringIO()
# redirect stdout to not get clobber file warnings
self.outfile = fitsio.FITS(self.outdir + self.outfilename, 'rw', clobber=True)
sys.stdout = old_stdout
dtype = self.infile[ext][0].dtype
input_header = fitsio.read_header(self.infilename, ext)
self.outfile.create_table_hdu(dtype=dtype, extname=input_header['EXTNAME'])
self.outfile[-1].write_keys(input_header)
self.outfile[-1]._update_info()
self.outfile.update_hdu_list()
# copy primary header from input to output file
primary_header = fitsio.read_header(self.infilename, 0)
self.outfile[0].write_keys(primary_header)
self.outfile[0].write_key('PIPE_VER', PIPELINE_VERSION, comment="GBT Pipeline Version")
return dtype
def __init__(self, config, depthfile=None):
"""
Instantiate a healpix-based redmapper depth map.
Parameters
----------
config: `redmapper.Configuration`
Configuration object, with config.depthfile set
depthfile: `str`, optional
Name of depthfile to use instead of config.depthfile
"""
# record for posterity
if depthfile is None:
self.depthfile = config.depthfile
else:
self.depthfile = depthfile
hdr = fitsio.read_header(self.depthfile, ext=1)
if 'PIXTYPE' not in hdr or hdr['PIXTYPE'] != 'HEALSPARSE':
raise RuntimeError(
"Need to specify depthfile in healsparse format. See redmapper_convert_depthfile_to_healsparse.py"
)
cov_hdr = fitsio.read_header(self.depthfile, ext='COV')
nside_coverage = cov_hdr['NSIDE']
self.nsig = cov_hdr['NSIG']
self.zp = cov_hdr['ZP']
self.nband = cov_hdr['NBAND']
self.w = cov_hdr['W']
self.eff = cov_hdr['EFF']
if config.d.hpix > 0:
covpixels = get_healsparse_subpix_indices(config.d.nside,
config.d.hpix,
config.border,
nside_coverage)
else:
covpixels = None
self.sparse_depthmap = healsparse.HealSparseMap.read(self.depthfile,
pixels=covpixels)
self.galfile_nside = config.galfile_nside
self.config_logger = config.logger
self.nside = self.sparse_depthmap.nsideSparse
self.config_area = config.area
# Record the coverage of the subregion that we read
self.subpix_nside = config.d.nside
self.subpix_hpix = config.d.hpix
self.subpix_border = config.border
def _load_source_image_info_fromdb(self):
"""
Load all meta information needed from the
ngmwint files
"""
# read the full coadd info that we dumped to file
fname=files.get_coaddinfo_file(
self['medsconf'],
self.file_dict['tilename'],
self.file_dict['band'],
)
print("reading full coaddinfo:",fname)
with open(fname) as fobj:
ci = yaml.load(fobj)
if self['source_type'] == 'nullwt':
# refined astrometry already present
entry='nullwt_path'
self['use_astro_refine']=False
else:
entry='image_path'
red_info=[]
for s in ci['src_info']:
path=s[entry]
sid = self._get_filename_as_id(path)
# now mock up the structure of the Coadd.srclist
if self['use_astro_refine']:
img_hdr = fitsio.read_header(path, ext=self['se_image_ext'])
wcs_hdr = fitsio.read_scamp_head(s['head_path'])
wcs_hdr = util.add_naxis_to_fitsio_header(wcs_hdr,img_hdr)
else:
wcs_hdr = fitsio.read_header(path, ext=self['se_image_ext'])
wcs_hdr = util.fitsio_header_to_dict(wcs_hdr)
s={
'id':sid,
'flags':0, # assume no problems!
'red_image':path,
'magzp':s['magzp'],
'wcs_header':wcs_hdr,
}
red_info.append(s)
return red_info
def _load_data(self, indir, expnum1, expnum2):
"""
Read in the two images, and check that headers are reasonable.
Sets self.bigimg[1,2] and returns True if everything is ok, else return False."""
filename1 = get_filename(indir, self.cam, expnum1)
filename2 = get_filename(indir, self.cam, expnum2)
if not os.path.exists(filename1) and not os.path.exists(filename2):
raise HartError('All files not found: %s, %s!' %
(filename1, filename2))
# NOTE: we don't process with sdssproc, because the subarrays cause it to fail.
# Also, because it would restore the dependency on SoS that this class removed!
try:
if fitsio:
header1 = fitsio.read_header(filename1, 0)
else:
header1 = pyfits.getheader(filename1, 0)
except IOError:
raise HartError('Failure reading file %s' % filename1)
try:
if fitsio:
header2 = fitsio.read_header(filename2, 0)
else:
header2 = pyfits.getheader(filename2, 0)
except IOError:
raise HartError('Failure reading file %s' % filename2)
if self.bad_header(header1) or self.bad_header(header2):
raise HartError('Incorrect header values in fits file.')
self.hartpos1 = self.check_Hartmann_header(header1)
self.hartpos2 = self.check_Hartmann_header(header2)
if self.hartpos1 is None or self.hartpos2 is None:
raise HartError(
'FITS headers do not indicate these are Hartmann exposures.')
if self.hartpos1 == self.hartpos2:
raise HartError(
'FITS headers indicate both exposures had same Hartmann position: %s'
% self.hartpos1)
# upcast the arrays, to make later math work better.
if fitsio:
self.bigimg1 = np.array(fitsio.read(filename1, 0), dtype='float64')
self.bigimg2 = np.array(fitsio.read(filename2, 0), dtype='float64')
else:
self.bigimg1 = np.array(pyfits.getdata(filename1, 0),
dtype='float64')
self.bigimg2 = np.array(pyfits.getdata(filename2, 0),
dtype='float64')
self.header1 = header1
self.header2 = header2
def get_wcs(self):
hdr = fitsio.read_header(self.imgfn, self.hdu)
wcs = wcs_pv2sip_hdr(hdr)
phdr = fitsio.read_header(self.imgfn, 0)
dra,ddec = self.decals.get_astrometric_zeropoint_for(self)
r,d = wcs.get_crval()
print('Applying astrometric zeropoint:', (dra,ddec))
wcs.set_crval((r + dra, d + ddec))
wcs.version = ''
wcs.plver = phdr.get('PLVER', '').strip()
return wcs
def get_wcs(self):
hdr = fitsio.read_header(self.imgfn, self.hdu)
wcs = wcs_pv2sip_hdr(hdr)
phdr = fitsio.read_header(self.imgfn, 0)
dra, ddec = self.decals.get_astrometric_zeropoint_for(self)
r, d = wcs.get_crval()
print('Applying astrometric zeropoint:', (dra, ddec))
wcs.set_crval((r + dra, d + ddec))
wcs.version = ''
wcs.plver = phdr.get('PLVER', '').strip()
return wcs
def _get_srclist(self):
"""
set the srclist, checking possibly for redone astrometry.
also check against blacklist
"""
srclist = self.cf.srclist
blacklists.add_bigind(srclist)
srclist = blacklists.remove_corrupted(srclist)
if len(srclist)==0:
raise RuntimeError("all src were in the corrupted list")
# do blacklists
for s in srclist:
s['flags'] = 0
blacklists.add_blacklist_flags(srclist)
# read astrom header
for s in srclist:
img_hdr = fitsio.read_header(s['red_image'],
ext=self['se_image_ext'])
if self['use_astro_refine']:
wcs_hdr = fitsio.read_scamp_head(s['astro_refine'])
wcs_hdr = util.add_naxis_to_fitsio_header(wcs_hdr,img_hdr)
else:
wcs_hdr = img_hdr
s['wcs_header'] = util.fitsio_header_to_dict(wcs_hdr)
return srclist
def read_sky_model(self, splinesky=False, slc=None):
'''
Reads the sky model, returning a Tractor Sky object.
'''
fn = self.skyfn
if splinesky:
fn = self.splineskyfn
print('Reading sky model from', fn)
hdr = fitsio.read_header(fn)
skyclass = hdr['SKY']
clazz = get_class_from_name(skyclass)
if getattr(clazz, 'from_fits', None) is not None:
fromfits = getattr(clazz, 'from_fits')
skyobj = fromfits(fn, hdr)
else:
fromfits = getattr(clazz, 'fromFitsHeader')
skyobj = fromfits(hdr, prefix='SKY_')
if slc is not None:
sy,sx = slc
x0,y0 = sx.start,sy.start
skyobj.shift(x0, y0)
skyobj.version = hdr.get('LEGPIPEV', '')
if len(skyobj.version) == 0:
skyobj.version = hdr.get('TRACTORV', '').strip()
if len(skyobj.version) == 0:
skyobj.version = str(os.stat(fn).st_mtime)
skyobj.plver = hdr.get('PLVER', '').strip()
return skyobj
def read_sky_model(self):
hdr = fitsio.read_header(self.skyfn)
skyclass = hdr['SKY']
clazz = get_class_from_name(skyclass)
fromfits = getattr(clazz, 'fromFitsHeader')
skyobj = fromfits(hdr, prefix='SKY_')
return skyobj
def _load_src_info_fromfile(self, finalcut_flist):
print('reading finalcut info from:',finalcut_flist)
print('using ohead files for the wcs')
src_info = []
with open(finalcut_flist) as fobj:
for line in fobj:
ls = line.split()
red_path=ls[0]
ohead_path=ls[1]
magzp=float(ls[2])
#magzp=30.0
img_hdr = fitsio.read_header(red_path, ext=self['se_image_ext'])
wcs_hdr = fitsio.read_scamp_head(ohead_path)
wcs_hdr = util.add_naxis_to_fitsio_header(wcs_hdr,img_hdr)
sid = self._get_filename_as_id(red_path)
entry = {
'id':sid,
'flags':0,
'red_image':red_path,
'magzp':magzp,
'wcs_header':wcs_hdr,
}
src_info.append(entry)
return src_info
def read_tan_wcs(sourcefn, ext, hdr=None, W=None, H=None, fitsfile=None):
from astrometry.util.util import Tan
if not os.path.exists(sourcefn):
return None
wcs = read_tansip_wcs(sourcefn, ext, hdr=hdr, W=W, H=H, tansip=Tan)
if wcs is None:
import fitsio
# maybe gzipped; try fitsio header.
if hdr is None:
hdr = fitsio.read_header(sourcefn, ext)
if W is None or H is None:
if fitsfile is None:
F = fitsio.FITS(sourcefn)
else:
F = fitsfile
info = F[ext].get_info()
H,W = info['dims']
# PS1 wonky WCS
if (not 'CD1_1' in hdr) and ('PC001001' in hdr):
cdelt1 = hdr['CDELT1']
cdelt2 = hdr['CDELT2']
# ????
cd11 = hdr['PC001001'] * cdelt1
cd12 = hdr['PC001002'] * cdelt1
cd21 = hdr['PC002001'] * cdelt2
cd22 = hdr['PC002002'] * cdelt2
else:
cd11,cd12,cd21,cd22 = hdr['CD1_1'], hdr['CD1_2'], hdr['CD2_1'], hdr['CD2_2'],
wcs = Tan(*[float(x) for x in [
hdr['CRVAL1'], hdr['CRVAL2'], hdr['CRPIX1'], hdr['CRPIX2'],
cd11, cd12, cd21, cd22, W, H]])
return wcs
def merge_partial_maps(filenames,outfile,**kwargs):
filenames = np.atleast_1d(filenames)
header = fitsio.read_header(filenames[0],ext=kwargs.get('ext',1))
nside = header['NSIDE']
data = ugali.utils.fileio.load_files(filenames,**kwargs)
pix = data['PIXEL']
ndupes = len(pix) - len(np.unique(pix))
if ndupes > 0:
msg = '%i duplicate pixels during load.'%(ndupes)
raise Exception(msg)
extname = 'DISTANCE_MODULUS'
distance = ugali.utils.fileio.load_files(filenames,ext=extname)[extname]
unique_distance = np.unique(distance)
# Check if distance moduli are the same...
if np.any(distance[:len(unique_distance)] != unique_distance):
msg = "Non-matching distance modulus:"
msg += '\n'+str(distance[:len(unique_distance)])
msg += '\n'+str(unique_distance)
raise Exception(msg)
write_partial_map(outfile,data=data,nside=nside,clobber=True)
fitsio.write(outfile,{extname:unique_distance},extname=extname)
def test_mask_write(self):
"""Test that masks are written to file correctly.
"""
# ADM some meaningless magnitude limits and mask epochs.
ml, me = 62.3, 2062.3
# ADM a keyword dictionary to write to the output file header.
extra = {'BLAT': 'blat', 'FOO': 'foo'}
# ADM test writing without HEALPixel-split.
_, mxdir = io.write_masks(self.maskdir, self.allmx, maglim=ml,
maskepoch=me, extra=extra)
# ADM test writing with HEALPixel-split.
_, mxdir = io.write_masks(self.maskdir, self.allmx, maglim=ml,
maskepoch=me, extra=extra, nside=self.nside)
# ADM construct the output directory and file name.
mxd = io.find_target_files(self.maskdir, flavor="masks",
maglim=ml, epoch=me)
mxfn = io.find_target_files(self.maskdir, flavor="masks",
maglim=ml, epoch=me, hp=self.pixnum[0])
# ADM check the output directory is as expected.
self.assertEqual(mxdir, mxd)
# ADM check all of the files were made in the correct place.
fns = glob(os.path.join(mxdir, "masks-hp*fits"))
self.assertEqual(len(fns), len(self.pixnum)+1)
# ADM check the extra kwargs were written to the header.
for key in extra:
hdr = fitsio.read_header(mxfn, "MASKS")
self.assertEqual(hdr[key].rstrip(), extra[key])
def read_dq(self, header=False, **kwargs):
from distutils.version import StrictVersion
print('Reading data quality from', self.dqfn, 'hdu', self.hdu)
dq,hdr = self._read_fits(self.dqfn, self.hdu, header=True, **kwargs)
# The format of the DQ maps changed as of version 3.5.0 of the
# Community Pipeline. Handle that here...
primhdr = fitsio.read_header(self.dqfn)
plver = primhdr['PLVER'].strip()
plver = plver.replace('V','')
if StrictVersion(plver) >= StrictVersion('3.5.0'):
dq = self.remap_dq_codes(dq)
else:
dq = dq.astype(np.int16)
# Un-set the SATUR flag for pixels that also have BADPIX set.
both = CP_DQ_BITS['badpix'] | CP_DQ_BITS['satur']
I = np.flatnonzero((dq & both) == both)
if len(I):
print('Warning: un-setting SATUR for', len(I),
'pixels with SATUR and BADPIX set.')
dq.flat[I] &= ~CP_DQ_BITS['satur']
assert(np.all((dq & both) != both))
if header:
return dq,hdr
else:
return dq
def filter(brickname, filename, region):
if not intersect(sweep, region):
return None, None
try:
objects = fitsio.read(filename, 1, upper=True)
chunkheader = fitsio.read_header(filename, 0, upper=True)
except:
if ns.ignore_errors:
print('IO error on %s' % filename)
return None, None
else:
raise
mask = objects['BRICK_PRIMARY'] != 0
objects = objects[mask]
mask = objects['RA'] >= ra1
mask &= objects['RA'] < ra2
mask &= objects['DEC'] >= dec1
mask &= objects['DEC'] < dec2
objects = objects[mask]
chunk = np.empty(len(objects), dtype=SWEEP_DTYPE)
for colname in chunk.dtype.names:
if colname not in objects.dtype.names:
# skip missing columns
continue
try:
chunk[colname][...] = objects[colname][...]
except ValueError:
print('failed on column `%s`' % colname)
raise
chunkheader = dict([(key, chunkheader[key]) for key in chunkheader.keys()])
return chunk, chunkheader
def read_intermediate_catalog(self, brick, **kwargs):
'''
Reads the intermediate tractor catalog for the given brickname.
*kwargs*: passed to self.find_file()
Returns (T, hdr, primhdr)
'''
fn = self.find_file('tractor-intermediate', brick=brick, **kwargs)
T = fits_table(fn)
hdr = T.get_header()
primhdr = fitsio.read_header(fn)
in_flux_prefix = ''
# Ensure flux arrays are 2d (N x 1)
keys = ['flux', 'flux_ivar', 'rchi2', 'fracflux', 'fracmasked',
'fracin', 'nobs', 'anymask', 'allmask', 'psfsize', 'depth',
'galdepth']
for k in keys:
incol = '%s%s' % (in_flux_prefix, k)
X = T.get(incol)
# Hmm, if we need to reshape one of these arrays, we will
# need to do all of them.
if len(X.shape) == 1:
X = X[:, np.newaxis]
T.set(incol, X)
return T, hdr, primhdr
def get_new_frame(night, expid, camera, basedir, nsky, rep=0):
'''For a given frame file, returns an updated frame file to the basedir with a certain number of sky fibers.
Args:
night: YYYYMMDD (float)
expid: exposure id without padding zeros (float)
camera: examples are b3, r4, z2, etc. (string)
basedir: path to directory you want new frame to be written
nsky: number of fibers flagged sky in new frame fibermap
Options:
rep: number of different frame files you want (with same number of sky fibers), default is 5
Writes a new frame file with desispec.io.write_frame().'''
framefile = desispec.io.findfile('frame', night, expid, camera=camera)
print(framefile)
header = fitsio.read_header(framefile)
fiberflatfile = findcalibfile([
header,
], 'FIBERFLAT')
frame = desispec.io.read_frame(framefile)
fiberflat = desispec.io.read_fiberflat(fiberflatfile)
print(fiberflatfile)
pick_sky_fibers(frame, fiberflat, nsky=nsky)
#- output updated frame to current directory
newframefile = basedir + '/frame-{}-{:08d}-{}-{}.fits'.format(
camera, expid, nsky, rep)
desispec.io.write_frame(newframefile, frame)
def run_sky_subtraction(night, expid, cameras, basedir, nsky_list, reps=None):
'''Runs sky subtraction with new sky models and frame files.
Args:
night: YYYYMMDD (float)
expid: exposure id without padding zeros (float)
cameras: list of cameras corresponding to frame and sky files in given directory, example ['r3', 'z3', 'b3'] (list or array)
basedir: where to look for frame files
nsky_list: list with different numbers of fibers frame files and sky files were generated with.
Options:
rep: number of different frame/sky files for each camera and nsky combination. Default is 5
'''
if reps == None:
reps = 5
for cam in cameras:
framefile = desispec.io.findfile('frame', night, expid, camera=cam)
header = fitsio.read_header(framefile)
fiberflatfile = findcalibfile([
header,
], 'FIBERFLAT')
fiberflat = desispec.io.read_fiberflat(fiberflatfile)
for N in range(reps):
for n in nsky_list:
newframefile = basedir + '/frame-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
skyfile = basedir + '/sky-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
sframe = desispec.io.read_frame(newframefile)
sky = desispec.io.read_sky(skyfile)
apply_fiberflat(sframe, fiberflat)
subtract_sky(sframe, sky)
sframefile = basedir + '/sframe-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
desispec.io.write_frame(sframefile, sframe)
def run_compute_sky(night, expid, cameras, basedir, nsky_list, reps=None):
'''Generates sky models for new frame files, using --no-extra-variance option, which doesn't inflate the output errors for sky subtraction systematics.
Args:
night: YYYYMMDD (float)
expid: exposure id without padding zeros (float)
cameras: list of cameras corresponding to frame files in given directory, example ['r3', 'z3', 'b3'] (list or array)
basedir: where to look for frame files
nsky_list: list with different numbers of fibers frame files were generated with.
Options:
rep: number of different frame files for each camera and nsky combination. Default is 5
'''
if reps == None:
reps = 5
for cam in cameras:
framefile = desispec.io.findfile('frame', night, expid, camera=cam)
header = fitsio.read_header(framefile)
fiberflatfile = findcalibfile([
header,
], 'FIBERFLAT')
for N in range(reps):
for n in nsky_list:
newframefile = basedir + '/frame-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
skyfile = basedir + '/sky-{}-{:08d}-{}-{}.fits'.format(
cam, expid, n, N)
cmd = 'desi_compute_sky -i {} --fiberflat {} -o {} --no-extra-variance'.format(
newframefile, fiberflatfile, skyfile)
print('RUNNING {}'.format(cmd))
err = subprocess.call(cmd.split())
if err:
print('FAILED')
else:
print('OK')
def getCards(fname, cards, hdu=0):
hdr = fitsio.read_header(fname, hdu)
ret = []
for c in cards:
ret.append(hdr[c])
return ret
def load_fiber_data(self, ext='FIBERASSIGN', hdr=False, simple=False):
"""
simple: Don't end up with dict['FIBERASSIGN'] = results.
"""
data = dict()
if isinstance(ext, list):
extlist = ext
else:
extlist = [ext]
for e in extlist:
data[e] = dict()
if hdr:
data['_HEADER'] = dict()
fiber_files = self.tile_files
print('Have {} tiles'.format(len(fiber_files)))
for fiber_file in fiber_files:
filename = os.path.basename(fiber_file)
itile = int(os.path.splitext(filename)[0].split('-')[1])
for e in extlist:
data[e][itile] = fitsio.read(fiber_file, e)
if hdr:
data['_HEADER'][itile] = fitsio.read_header(fiber_file)
if simple:
if len(extlist) == 1 and not hdr:
data = data[e]
return data
def merge_likelihood_headers(filenames, outfile, **kwargs):
"""
Merge header information from likelihood files.
Parameters:
-----------
filenames : input filenames
oufile : the merged file to write
Returns:
--------
data : the data being written
"""
import fitsio
filenames = np.atleast_1d(filenames)
ext = 'PIX_DATA'
nside = fitsio.read_header(filenames[0], ext=ext)['LKDNSIDE']
keys = ['LKDPIX', 'STELLAR', 'NINSIDE', 'NANNULUS']
data_dict = fileio.load_headers(filenames, ext=ext, keys=keys, mulitproc=8)
names = data_dict.dtype.names
data_dict.dtype.names = ['PIXEL' if n == 'LKDPIX' else n for n in names]
write_partial_map(outfile, data_dict, nside)
return data_dict
def _get_srclist(self):
"""
set the srclist, checking possibly for redone astrometry.
also check against blacklist
"""
srclist = self.cf.srclist
blacklists.add_bigind(srclist)
srclist = blacklists.remove_corrupted(srclist)
if len(srclist)==0:
raise RuntimeError("all src were in the corrupted list")
# do blacklists
for s in srclist:
s['flags'] = 0
blacklists.add_blacklist_flags(srclist)
# read astrom header
for s in srclist:
img_hdr = fitsio.read_header(s['red_image'],
ext=self['se_image_ext'])
if self['use_astro_refine']:
wcs_hdr = fitsio.read_scamp_head(s['astro_refine'])
wcs_hdr = util.add_naxis_to_fitsio_header(wcs_hdr,img_hdr)
else:
wcs_hdr = img_hdr
s['wcs_header'] = util.fitsio_header_to_dict(wcs_hdr)
return srclist
def main():
spec_names = os.listdir()
spec_names.sort()
file_list = open('list.txt', 'w')
file_list.write("FileName" + '\t' + "ImageType" + '\t' + "Date" + '\t' +
"Starting" + '\t' + "Explosure" + '\t' + "MiddleTime" +
'\t' + "HJD" + '\n')
for file in spec_names:
if file.endswith('.fit'):
# Reading FITS header
current = fitsio.read_header(file)
imagetyp = current['IMAGETYP']
date = current['DATE']
exp = current['EXPTIME']
# Some types have no some properties
if imagetyp == 'zero':
file_list.write(file + '\t' + imagetyp + '\t' + date + '\n')
continue
time = current['UT']
ra = current['RA']
dec = current['DEC']
jd = current['JDMID']
middle = middleTime(time, exp)
hjd = heliocentricJD(jd, ra, dec)
file_list.write(file + '\t' + imagetyp + '\t' + date + '\t' +
time + '\t' + str(exp) + '\t' + middle + ' ' +
str(hjd) + '\n')
file_list.close()
def _load_nwgint_info(self):
"""
Load all meta information needed from the
ngmwint files
"""
fname=self.file_dict['nwgint_flist']
print("reading nwgint list and loading headers:",fname)
red_info=[]
with open(fname) as fobj:
for line in fobj:
path, magzp = self._extract_nwgint_line(line)
if path==None:
continue
sid = self._get_filename_as_id(path)
# now mock up the structure of the Coadd.srclist
wcs_hdr = fitsio.read_header(path, ext=self['se_image_ext'])
wcs_header = util.fitsio_header_to_dict(wcs_hdr)
s={
'id':sid,
'flags':0, # assume no problems!
'red_image':path,
'magzp':magzp,
'wcs_header':wcs_header,
}
red_info.append(s)
return red_info
def get_sdss_dr7_frame(run, camcol, field, band, rerun=40):
url, psfield_url, calib_url = \
get_sdss_dr7_frame_url(run, camcol, field, band, rerun)
# download files if necessary
import wget
if not os.path.exists(img_filename):
img_filename = wget.download(url)
if not os.path.exists(ps_filename):
ps_filename = wget.download(psfield_url)
# load calibration data to get sky noise
ps_data = fitsio.FITS(ps_filename)[6].read()
# create fitsfile
img_data = fitsio.FITS(img_filename)[0].read()
img_header = fitsio.read_header(img_filename)
import CelestePy.fits_image as fits_image
reload(fits_image)
imgfits = fits_image.FitsImage(band,
timg=imgs[band],
calib=1.,
gain=gain,
darkvar=darkvar,
sky=0.)
def read_sky_model(self, splinesky=False, slc=None, **kwargs):
'''
Reads the sky model, returning a Tractor Sky object.
'''
fn = self.skyfn
if splinesky:
fn = self.splineskyfn
print('Reading sky model from', fn)
hdr = fitsio.read_header(fn)
skyclass = hdr['SKY']
clazz = get_class_from_name(skyclass)
if getattr(clazz, 'from_fits', None) is not None:
fromfits = getattr(clazz, 'from_fits')
skyobj = fromfits(fn, hdr)
else:
fromfits = getattr(clazz, 'fromFitsHeader')
skyobj = fromfits(hdr, prefix='SKY_')
if slc is not None:
sy, sx = slc
x0, y0 = sx.start, sy.start
skyobj.shift(x0, y0)
skyobj.version = hdr.get('LEGPIPEV', '')
if len(skyobj.version) == 0:
skyobj.version = hdr.get('TRACTORV', '').strip()
if len(skyobj.version) == 0:
skyobj.version = str(os.stat(fn).st_mtime)
skyobj.plver = hdr.get('PLVER', '').strip()
sig1 = hdr.get('SIG1', None)
if sig1 is not None:
skyobj.sig1 = sig1
return skyobj
def read_fits(path):
"""
Read fits file and return gmm parameters
"""
fits_info = fitsio.read_header(path)
gfa_names = [
"GUIDE0", "GUIDE2", "GUIDE3",
"GUIDE5", "GUIDE7", "GUIDE8"]
data = dict()
data["expinfo"] = {"night": fits_info["NIGHT"],
"expid": fits_info["EXPID"],
"MJD": fits_info["MJD-OBS"]}
data["GUIDE"] = dict()
psf_pixels = 25
psf_grid = np.arange(psf_pixels + 1) - psf_pixels / 2
GMM = desietc.gmm.GMMFit(psf_grid, psf_grid)
with fitsio.FITS(path, mode="r") as hdus:
for gfa in gfa_names:
if gfa not in hdus: continue
gmm_params = fitsio.read(path, ext=gfa+"M")
psf = GMM.predict(gmm_params)
psf[psf < 0.0] = 0.0
data["GUIDE"][gfa] = {"model": psf}
return data
def run_assemble_fibermap(rawfile, outdir):
'''Run assemble_fibermap using NIGHT, EXPID, and TILE from input raw data file
Args:
rawfile: input desi-EXPID.fits.fz raw data file
outdir: directory to write fibermap-EXPID.fits files
Returns:
path to written fibermap
'''
hdr = fitsio.read_header(rawfile, 1)
night, expid = get_night_expid_header(hdr)
log = desiutil.log.get_logger()
if 'TILEID' in hdr:
if not os.path.isdir(outdir):
log.info('Creating {}'.format(outdir))
os.makedirs(outdir, exist_ok=True)
fibermap = os.path.join(outdir, 'fibermap-{:08d}.fits'.format(expid))
cmd = f'assemble_fibermap -n {night} -e {expid} -o {fibermap} --overwrite'
logfile = '{}/assemble_fibermap-{:08d}.log'.format(outdir, expid)
msg = 'assemble_fibermap {}/{}'.format(night, expid)
err = runcmd(cmd, logfile, msg)
return fibermap
return None
def from_fnames(filenames=['kplr2009114174833_ffi-cal.fits'],
error_filenames=['kplr2009114174833_ffi-uncert.fits'],
skygroup=70,
**kwargs):
if not (len(filenames) == len(error_filenames)):
raise ValueError(
'Length of input filenames and input error filenames are not the same'
)
extensions = np.zeros(len(filenames), dtype=int)
for idx, fname in enumerate(filenames):
extensions[idx] = np.where(
np.asarray([
fitsio.read_header(fname, ext=idx)['SKYGROUP']
for idx in np.arange(1, 85)
], int) == skygroup)[0][0] + 1
data = lazy_loader(filenames, extensions=extensions, name='data')
error = lazy_loader(error_filenames,
extensions=extensions,
name='error')
wcs = [WCS(hdr) for hdr in data.hdrs]
if not np.in1d(error.time, data.time).all():
raise ValueError(
'Not all times are identical between input filenames and input error filenames'
)
return Cube(data, error, wcs=wcs, **kwargs)
def read_map2(filename,field=0,dtype=np.float64,nest=False,hdu=1,h=False,verbose=True,memmap=False):
hdr=fitsio.read_header(filename,ext=hdu)
fullsky = False
try:
if (hdr['OBJECT'].strip() == 'PARTIAL') :
# partial sky format
fullsky=False
else:
fullsky=True
except:
# if no OBJECT in header, assume full sky
fullsky=True
if fullsky:
m=hp.read_map(filename,field=field,dtype=dtype,nest=nest,hdu=hdu,h=h,verbose=verbose,memmap=memmap)
else:
# partial sky
st=fitsio.read(filename,ext=1)
nside=hdr['NSIDE']
m=np.zeros(12*nside*nside,dtype=dtype) + hp.UNSEEN
if ((hdr['ORDERING'].strip() == 'NESTED') and (not nest)) :
# change from nest to ring...
m[hp.nest2ring(nside,st['PIXEL'])] = st['SIGNAL']
elif ((hdr['ORDERING'].strip() == 'RING') and (nest)):
# change from ring to nest...
m[hp.ring2nest(nside,st['PIXEL'])] = st['SIGNAL']
else :
# straight up
m[st['PIXEL']] = st['SIGNAL']
return m
def main():
#fns = glob('/global/project/projectdirs/cosmo/work/legacysurvey/dr8/decam/tractor/000/tractor-000??00?.fits')
fns = glob(
'/global/project/projectdirs/cosmo/work/legacysurvey/dr8/decam/tractor/*/tractor-*.fits'
)
fns.sort()
print(len(fns), 'Tractor catalogs')
vers = Counter()
keepfns = []
for fn in fns:
hdr = fitsio.read_header(fn)
ver = hdr['LEGPIPEV']
ver = ver.strip()
vers[ver] += 1
if ver == 'DR8.2.1':
keepfns.append(fn)
print('Header versions:', vers.most_common())
fns = keepfns
print('Keeping', len(fns), 'with bad version')
N = len(fns)
args = [(i, N, fn) for i, fn in enumerate(fns)]
mp = multiproc(8)
mp.map(patch_one, args)
def get_sdss_dr7_frame(run, camcol, field, band, rerun=40):
url, psfield_url, calib_url = \
get_sdss_dr7_frame_url(run, camcol, field, band, rerun)
# download files if necessary
import wget
if not os.path.exists(img_filename):
img_filename = wget.download(url)
if not os.path.exists(ps_filename):
ps_filename = wget.download(psfield_url)
# load calibration data to get sky noise
ps_data = fitsio.FITS(ps_filename)[6].read()
# create fitsfile
img_data = fitsio.FITS(img_filename)[0].read()
img_header = fitsio.read_header(img_filename)
import CelestePy.fits_image as fits_image
reload(fits_image)
imgfits = fits_image.FitsImage(band,
timg=imgs[band],
calib=1.,
gain=gain,
darkvar=darkvar,
sky=0.)
def test_header(self):
myhdr = self.hdu.hdr
hdr = fitsio.read_header(self.path)
self.assertEqual(myhdr['NAXIS'], hdr['NAXIS'])
self.assertEqual(myhdr['NAXIS3'], hdr['NAXIS3'])
self.assertEqual(myhdr['BITPIX'], hdr['BITPIX'])
def skyplot():
plot = Plotstuff(size=(800, 800), rdw=(103.1, 37.45, 0.8), outformat='png')
plot.color = 'verydarkblue'
plot.plot('fill')
for ext in range(1, 17):
fn = 'mos3.68488.fits'
hdr = fitsio.read_header(fn, ext=ext)
wcs = wcs_pv2sip_hdr(hdr)
plot.color = 'red'
plot.outline.wcs = anwcs_new_sip(wcs)
plot.plot('outline')
plot.color = 'white'
plot.apply_settings()
rc, dc = wcs.radec_center()
plot.text_radec(rc, dc, hdr['EXTNAME'])
plot.color = 'white'
for ext in range(1, 17):
fn = 'an2/mos3.68488.ext%02i.wcs' % ext
plot.outline.wcs = anwcs(fn)
plot.plot('outline')
plot.rgb = (0.2, 0.2, 0.2)
plot.plot_grid(0.1, 0.1)
plot.color = 'gray'
plot.plot_grid(0.5, 0.5, 0.5, 0.5)
plot.write('plot.png')
def get_all(self, files):
"""Get the headers from all requested files."""
result = []
for f in files:
frameno = int(f.split('-')[-1].split('.')[0])
header = fitsio.read_header(f, 0)
#header = pyfits.getheader(f)
# don't want darks or flats
if 'object' in header['IMAGETYP']:
if self.verbose:
print "Extracting:", f
line = self.get_one(header)
line.insert(0, frameno)
result.append(line)
else:
if self.verbose:
print "Ignoring:", f
del header
names = [k[0] for k in self.keys]
types = [k[1] for k in self.keys]
for i, t in enumerate(types):
if t == str:
types[i] = '|S40'
names.insert(0, 'FRAMENO')
types.insert(0, int)
# self.data = np.rec.fromrecords(result,dtype=np.dtype(zip(names,types)))
# self.data = np.array(result,dtype=np.dtype(zip(names,types)))
self.data = Table(rows=result, names=names, dtype=types)
def _get_box_sizes(self, image_info, cat):
"""
get box sizes that are wither 2**N or 3*2**N, within
the limits set by the user
"""
file_id = 0
impath = image_info['image_path'][file_id].strip()
ext = image_info['image_ext'][file_id]
wcs_data = fitsio.read_header(impath, ext=ext)
wcs = eu.wcsutil.WCS(wcs_data)
jacob = wcs.get_jacobian(100, 100)
dudcol, dudrow, dvdcol, dvdrow = jacob
det = dvdrow * dudcol - dvdcol * dudrow
pixel_scale = np.sqrt(abs(det))
print('found pixel scale:', pixel_scale)
box_size = cat['box_size_arcsec'] / pixel_scale
# clip to range
box_size.clip(
min=self['min_box_size'],
max=self['max_box_size'],
out=box_size,
)
box_size = box_size.astype('i4')
w, = np.where(((box_size % 2) != 0))
if w.size > 0:
box_size[w] += 1
return box_size
def load_fiber_data_lean(path, ext='FIBERASSIGN', hdr=False):
"""
"""
data = dict()
if isinstance(ext, list):
extlist = ext
else:
extlist = [ext]
for e in extlist:
data[e] = dict()
if hdr:
data['_HEADER'] = dict()
fiber_files = glob_fibers(path)
print('Have {} tiles'.format(len(fiber_files)))
for fiber_file in fiber_files:
filename = os.path.basename(fiber_file)
itile = int(os.path.splitext(filename)[0].split('-')[1])
for e in extlist:
data[e][itile] = fitsio.read(fiber_file, e)
if hdr:
data['_HEADER'][itile] = fitsio.read_header(fiber_file)
return data
def write_astrometry_for_gfa_file(fn,
out_dir,
gnums=[0, 2, 3, 5, 7, 8],
left=False,
config_fn=None):
hdr = fitsio.read_header(fn, ext=1)
skyra = hdr['SKYRA']
skydec = hdr['SKYDEC']
expnum = hdr['EXPID']
imgfns = []
crpix_arg = '--crpix-center'
for gnum in gnums:
gname = 'GUIDE%i' % gnum
g, x0, y0 = read_guide_image_file(fn, ext=gname)
good = read_good_pix_maps(gname)
imgfn = os.path.join(out_dir, 'gfa-%i-%s.fits' % (expnum, gname))
g = g * good
gh, gw = g.shape
if left:
g = g[:, :gw // 2]
crpix_arg = '--crpix-x %.1f --crpix-y %.1f' % (gw + 0.5,
gh / 2 + 0.5)
fitsio.write(imgfn, g, clobber=True)
imgfns.append(imgfn)
if out_dir == '':
out_dir = '.'
if config_fn is None:
config_fn = commish_paths.an_config_filename
# NOTE -- definitely want --tweak-order 1, otherwise (with
# --tweak-order 0 or --no-tweak) we get a SQUARE CD matrix!!
cmd = ''
an_dir = commish_paths.an_path
if an_dir is not None:
cmd = ('PATH=%s/bin:${PATH} ' % an_dir) + cmd
an_py_path = commish_paths.an_py_path
if an_py_path is not None:
cmd = ('PYTHONPATH=%s:${PYTHONPATH} ' % an_py_path) + cmd
cmd += ((
'solve-field --config %s --xscale 1.1' +
' --ra %f --dec %f --radius 2 --scale-low 0.18 --scale-high 0.24 --scale-units app --downsample 2'
+ ' --continue --tweak-order 1 --plot-scale 0.5 --objs 100' +
' --batch --dir %s %s ') %
(config_fn, skyra, skydec, out_dir, crpix_arg))
cmd += ' '.join(imgfns)
print(cmd)
os.system(cmd)
wcsfns = [fn.replace('.fits', '.wcs') for fn in imgfns]
wcsfns = [fn if os.path.exists(fn) else None for fn in wcsfns]
return wcsfns, hdr
def run(self, indir):
'''TODO: document'''
log = desiutil.log.get_logger()
infiles = glob.glob(os.path.join(indir, 'psf-*.fits'))
results = list()
for filename in infiles:
log.debug(filename)
hdr = fitsio.read_header(filename)
night = hdr['NIGHT']
expid = hdr['EXPID']
cam = hdr['CAMERA'][0].upper()
spectro = int(hdr['CAMERA'][1])
dico={"NIGHT":night,"EXPID":expid,"SPECTRO":spectro,"CAM":cam}
xsig = fitsio.read(filename,"XSIG")
ysig = fitsio.read(filename,"XSIG")
dico["MEANXSIG"]=np.mean(xsig[:,0])
dico["MINXSIG"]=np.min(xsig[:,0])
dico["MAXXSIG"]=np.max(xsig[:,0])
dico["MEANYSIG"]=np.mean(ysig[:,0])
dico["MINYSIG"]=np.min(ysig[:,0])
dico["MAXYSIG"]=np.max(ysig[:,0])
log.debug("{} {} {} {} mean xsig={:3.2f} ysig={:3.2f}".format(night,expid,cam,spectro,dico["MEANXSIG"],dico["MEANYSIG"]))
results.append(collections.OrderedDict(**dico))
return Table(results, names=results[0].keys())
def __init__(self, config, depthfile=None):
"""
Instantiate a healpix-based redmapper depth map.
Parameters
----------
config: `redmapper.Configuration`
Configuration object, with config.depthfile set
depthfile: `str`, optional
Name of depthfile to use instead of config.depthfile
"""
# record for posterity
if depthfile is None:
self.depthfile = config.depthfile
else:
self.depthfile = depthfile
hdr = fitsio.read_header(self.depthfile, ext=1)
if 'PIXTYPE' not in hdr or hdr['PIXTYPE'] != 'HEALSPARSE':
raise RuntimeError("Need to specify depthfile in healsparse format. See redmapper_convert_depthfile_to_healsparse.py")
cov_hdr = fitsio.read_header(self.depthfile, ext='COV')
nside_coverage = cov_hdr['NSIDE']
self.nsig = cov_hdr['NSIG']
self.zp = cov_hdr['ZP']
self.nband = cov_hdr['NBAND']
self.w = cov_hdr['W']
self.eff = cov_hdr['EFF']
if config.d.hpix > 0:
covpixels = get_healsparse_subpix_indices(config.d.nside, config.d.hpix,
config.border, nside_coverage)
else:
covpixels = None
self.sparse_depthmap = healsparse.HealSparseMap.read(self.depthfile, pixels=covpixels)
self.galfile_nside = config.galfile_nside
self.config_logger = config.logger
self.nside = self.sparse_depthmap.nsideSparse
self.config_area = config.area
# Record the coverage of the subregion that we read
self.subpix_nside = config.d.nside
self.subpix_hpix = config.d.hpix
self.subpix_border = config.border
def _build_object_data(self):
"""
make the object data such as box sizes and ra,dec based on
the row,col->ra,dec transformation
"""
print('building basic object data')
nobj=len(self.coadd_cat)
extra_fields=self['extra_obj_data_fields']
self.obj_data = get_meds_input_struct(nobj,
extra_fields=extra_fields)
eu.numpy_util.copy_fields(
self.coadd_cat,
self.obj_data,
)
#self.obj_data['number'] = self.coadd_cat['number']
input_row = self.coadd_cat[self['row_name']]
input_col = self.coadd_cat[self['col_name']]
pos=self._make_wcs_positions(input_row, input_col)
self.obj_data['input_row'] = pos['zrow']
self.obj_data['input_col'] = pos['zcol']
self.obj_data['flags'] = self.coadd_cat[self['flags_name']]
self.obj_data['flux'] = self.coadd_cat[self['flux_name']]
self.obj_data['flux_err'] = self.coadd_cat[self['fluxerr_name']]
self.obj_data['x2'] = self.coadd_cat[self['x2_name']]
self.obj_data['x2_err'] = self.coadd_cat[self['x2err_name']]
self.obj_data['y2'] = self.coadd_cat[self['y2_name']]
self.obj_data['y2_err'] = self.coadd_cat[self['y2err_name']]
iso_area = self.coadd_cat[self['isoarea_name']].clip(min=1)
self.obj_data['iso_radius'] = sqrt(iso_area/PI)
# required
self.obj_data['box_size'] = self._get_box_sizes()
# do coadd ids and check things
iddata = self._get_coadd_objects_ids()
q = numpy.argsort(iddata['object_number'])
iddata = iddata[q]
mess="Could not find all objects in DESDM table!"
check=numpy.array_equal(self.coadd_cat['number'],iddata['object_number'])
assert check,mess
# required
self.obj_data['id'] = iddata['coadd_objects_id']
# get ra,dec
coadd_hdr = fitsio.read_header(self.cf_refband['image_url'],
ext=self['coadd_image_ext'])
coadd_wcs = eu.wcsutil.WCS(coadd_hdr)
ra,dec = coadd_wcs.image2sky(pos['wcs_col'], pos['wcs_row'])
self.obj_data['ra'] = ra
self.obj_data['dec'] = dec
def extract_data(self):
self.header = fitsio.read_header(self.fname)
extracted = {
key: getattr(self, key) for key in self.keys
}
extracted.update(self.wcs_solution())
return extracted
def read_pv_wcs(self):
'''extract wcs from fits header directly'''
hdr = fitsio.read_header(self.imgfn, self.hdu)
H,W = self.get_image_shape()
wcs= Tan(hdr['CRVAL1'], hdr['CRVAL2'],hdr['CRPIX1'],hdr['CRPIX2'],\
hdr['CD1_1'],hdr['CD1_2'],hdr['CD2_1'],hdr['CD2_2'],\
float(W),float(H))
return wcs
def upload_cat(req):
import tempfile
from astrometry.util.fits import fits_table
from django.http import HttpResponseRedirect
from map.views import index
if req.method != 'POST':
return HttpResponse('POST only')
print('Files:', req.FILES)
cat = req.FILES['catalog']
dirnm = settings.USER_QUERY_DIR
if not os.path.exists(dirnm):
try:
os.makedirs(dirnm)
except:
pass
f,tmpfn = tempfile.mkstemp(suffix='.fits', dir=dirnm)
os.close(f)
os.unlink(tmpfn)
print('Saving to', tmpfn)
with open(tmpfn, 'wb+') as destination:
for chunk in cat.chunks():
destination.write(chunk)
print('Wrote', tmpfn)
try:
T = fits_table(tmpfn)
except:
return HttpResponse('Must upload FITS format catalog including "RA", "Dec", optionally "Name" columns')
# Rename and resave columns if necessary
if rename_cols(T):
T.write_to(tmpfn)
cols = T.columns()
if not (('ra' in cols) and ('dec' in cols)):
return HttpResponse('Must upload catalog including "RA", "Dec", optionally "Name" columns')
ra,dec = T.ra[0], T.dec[0]
catname = tmpfn.replace(dirnm, '').replace('.fits', '')
if catname.startswith('/'):
catname = catname[1:]
try:
import fitsio
primhdr = fitsio.read_header(tmpfn)
name = primhdr.get('CATNAME', '')
color = primhdr.get('CATCOLOR', '')
if len(name):
catname = catname + '-n%s' % name.strip().replace(' ','_')
if len(color):
catname = catname + '-c%s' % color.strip()
except:
pass
return HttpResponseRedirect(reverse(index) +
'?ra=%.4f&dec=%.4f&catalog=%s' % (ra, dec, catname))
def extract_meta(filename):
header = fitsio.read_header(filename)
return {
'image_id': header.get('IMAGE_ID', None),
'gain': header.get('GAINFACT', None),
'gain_setting': header.get('GAIN', None),
'camera_id': header.get('CAMERAID', None),
'vi+': header.get('VI_PLUS', None),
}
def read_image_primary_header(self, **kwargs):
'''
Reads the FITS primary (HDU 0) header from self.imgfn.
Returns
-------
primary_header : fitsio header
The FITS header
'''
return fitsio.read_header(self.imgfn)
def read_image_header(self, **kwargs):
'''
Reads the FITS image header from self.imgfn HDU self.hdu.
Returns
-------
header : fitsio header
The FITS header
'''
return fitsio.read_header(self.imgfn, ext=self.hdu)
def read_equirectangular(file_name, unit_scale=1, ):
f = fit.read_header(file_name)
data = fit.read(file_name)[0]
data[:, :-1] = np.roll(data[:, :-1], (f['NAXIS1'] - 1) / 2, axis=1)[:, ::-1]
data[:, -1] = data[:, 0]
dmax = f['DATAMAX']
dmin = f['DATAMIN']
data[data > dmax] = -1e6 * dmax
data[data < dmin] = -1e6 * dmax
data = sv.equirectangular2heapix(data)
data[data <= 0] = np.nan
return unit_scale * data
def __init__(self, galaxy, scale, band):
self.topdir = os.path.join(os.getenv('HIZEA_DATA'),'hst','aplus','cutouts')
self.imfile = os.path.join(self.topdir,'{}-{}-{}.fits'.format(galaxy,scale,band))
self.ivarfile = os.path.join(self.topdir,'{}-{}-{}-ivar.fits'.format(galaxy,scale,band))
self.psffile = os.path.join(self.topdir,'{}-{}-{}.psf'.format(galaxy,scale,band))
print('Reading image {}'.format(self.imfile))
self.image = fitsio.read(self.imfile, ext=0)
self.header = fitsio.read_header(self.imfile)
print('Reading image {}'.format(self.ivarfile))
self.ivar = fitsio.read(self.ivarfile, ext=0)
def fnote(fname, ftype='', notes=''):
if fname is None:
return
hdr = fitsio.read_header(fname)
hdrNotes = "ccd=%s,%s pa=%s %s %s" % (hdr.get('temp.ccd0'),
hdr.get('temp.ccd1'),
hdr.get('temp.PA'),
hdr.get('IMAGETYP', 'unknown').strip(),
hdr.get('EXPTIME', ''))
note("%s %s %s %s" % (fname, ftype, hdrNotes, notes))
def _get_wcs_json(self,srclist):
"""
get string versions of the wcs for each image
"""
coadd_wcs = fitsio.read_header(self.cf['image_url'],
ext=self['coadd_image_ext'])
wcs_json = []
wcs_json.append(json.dumps(util.fitsio_header_to_dict(coadd_wcs)))
for s in srclist:
#wcs_json.append(json.dumps(s['wcs_header']))
wcs_json.append(json.dumps(util.fitsio_header_to_dict(s['wcs_header'])))
return wcs_json
def correct_catfile(catfile_name, image_name, nstars=False):
with fitsio.FITS(catfile_name, 'rw') as cat:
im_header = fitsio.read_header(image_name)
my_X = cat[1]['X_coordinate'][:]
old_RA = cat[1]['RA'][:]
my_Y = cat[1]['Y_coordinate'][:]
pix_coords = [[my_X[i], my_Y[i]] for i in range(0, len(my_X))]
world = load_wcs_from_keywords(im_header, pix_coords)
RA = world[:, 0] * np.pi / 180
DEC = world[:, 1] * np.pi / 180
cat[1].write_column('RA', RA, clobber=True)
cat[1].write_column('DEC', DEC, clobber=True)
def fromFits(fn):
from astrometry.util.fits import fits_table
import fitsio
hdr = fitsio.read_header(fn, ext=1)
T = fits_table(fn)
assert(len(T) == 1)
#for col in T.get_columns():
# if col.strip() != col:
# T.rename_column(col, col.strip())
T = T[0]
t = hdr['PSFEX_T'].strip()
#print 'Type:', t
assert(t == 'tractor.psfex.PsfEx')
psft = hdr['PSF_TYPE']
knowntypes = dict([(typestring(x), x)
for x in [GaussianMixturePSF,
GaussianMixtureEllipsePSF]])
psft = knowntypes[psft]
#print 'PSF type:', psft
nx = hdr['PSF_NX']
ny = hdr['PSF_NY']
w = hdr['PSF_W']
h = hdr['PSF_H']
K = hdr['PSF_K']
psfex = PsfEx(None, w, h, nx=nx, ny=ny, K=K, psfClass=psft)
nargs = hdr['PSF_NA']
pp = np.zeros((ny,nx,nargs))
columns = T.get_columns()
for i in range(nargs):
nm = hdr['PSF_A%i' % i].strip()
#print 'param name', nm
if nm in columns:
pi = T.get(nm)
assert(pi.shape == (ny,nx))
pp[:,:,i] = pi
else:
# param name like "amp0"
assert(nm[-1] in '0123456789'[:K])
pi = T.get(nm[:-1])
#print 'array shape', pi.shape
assert(pi.shape == (K,ny,nx))
k = int(nm[-1], 10)
pi = pi[k,:,:]
#print 'cut shape', pi.shape
assert(pi.shape == (ny,nx))
pp[:,:,i] = pi
psfex.splinedata = (pp, T.xx, T.yy)
return psfex
def make_zred_table(self, indices_and_filenames):
"""
Make a zred table from a list of indices and filenames
Saves to self.config.zredfile.
Parameters
----------
indices_and_filenames: `list`
List of (index, outfile) tuples describing zred files.
"""
# figure out longest filename
maxlen = 0
for index, filename in indices_and_filenames:
if len(os.path.basename(filename)) > maxlen:
maxlen = len(os.path.basename(filename))
gal_dtype = self.galtable.dtype
zred_dtype = []
for dt in gal_dtype.descr:
if dt[0].lower() == 'filenames':
dt = ('filenames', 'S%d' % (maxlen + 1), dt[2])
zred_dtype.append(dt)
zredtable = Entry(np.zeros(1,dtype=zred_dtype))
for name in gal_dtype.names:
zredtable._ndarray[name] = self.galtable._ndarray[name]
zredtable.filenames = ''
zredtable.ngals = 0
for index, filename in indices_and_filenames:
# Make sure file exists
if not os.path.isfile(filename):
raise ValueError("Could not find zredfile: %s" % (filename))
# check size of file
hdr = fitsio.read_header(filename, ext=1)
if hdr['NAXIS2'] != self.galtable.ngals[index]:
raise ValueError("Length mismatch for zredfile: %s" % (filename))
zredtable.filenames[index] = os.path.basename(filename)
zredtable.ngals[index] = self.galtable.ngals[index]
hdr = fitsio.FITSHDR()
hdr['PIXELS'] = 1
zredtable.to_fits_file(self.config.zredfile, header=hdr, clobber=True)
def __init__(self, fn=None, ext=1):
'''
`ext` is ignored.
'''
if fn is not None:
from astrometry.util.fits import fits_table
T = fits_table(fn)
T.about()
ims = fitsio.read(fn, ext=2)
print('Eigen-images', ims.shape)
nsch,h,w = ims.shape
hdr = fitsio.read_header(fn)
x0 = 0.
y0 = 0.
xscale = 1. / hdr['XSCALE']
yscale = 1. / hdr['YSCALE']
degree = (T.xexp + T.yexp).max()
self.sampling = 1.
# Reorder the 'ims' to match the way PsfEx sorts its polynomial terms
# number of terms in polynomial
ne = (degree + 1) * (degree + 2) / 2
print('Number of eigen-PSFs required for degree=', degree, 'is', ne)
self.psfbases = np.zeros((ne, h,w))
for d in range(degree + 1):
# x polynomial degree = j
# y polynomial degree = k
for j in range(d+1):
k = d - j
ii = j + (degree+1) * k - (k * (k-1))/ 2
jj = np.flatnonzero((T.xexp == j) * (T.yexp == k))
if len(jj) == 0:
print('Schlegel image for power', j,k, 'not found')
continue
im = ims[jj,:,:]
print('Schlegel image for power', j,k, 'has range', im.min(), im.max(), 'sum', im.sum())
self.psfbases[ii,:,:] = ims[jj,:,:]
self.xscale, self.yscale = xscale, yscale
self.x0,self.y0 = x0,y0
self.degree = degree
print('SchlegelPsfEx degree:', self.degree)
bh,bw = self.psfbases[0].shape
self.radius = (bh+1)/2.