def bin_run(run, bin_conf_name, jackreg_col=None):
"""
Do the binning and write out a file
"""
import fitsio
data = read_collated(run)
binner=Binner(bin_conf_name)
res = binner.bin(data, jackreg_col=jackreg_col)
if jackreg_col is None:
fname=get_binned_file(run, bin_conf_name)
d=get_binned_dir(run, bin_conf_name)
else:
fname=get_jack_file(run, bin_conf_name)
d=get_jack_dir(run, bin_conf_name)
if not os.path.exists(d):
print("Making dir:",d)
os.makedirs(d)
print("writing:",fname)
fitsio.write(fname, res, clobber=True)
for i in xrange(binner.get_nbin()):
plt=binner.plot_radec(data,i)
epsname=fname.replace('.fits','-%02d-radec.eps' % i)
pngname=epsname.replace('.eps','.png')
print(pngname)
plt.write_img(1000,1000,pngname)
command='cd %s; im2html -p *.png > radec.html' % d
print("making html file")
os.system(command)
def cc_restore_stars(self, algorithm):
"""
Restores the cookie cut regions back to the compressed image with the
ORIGINAL pixel values.
@type self: SuperBit_Compression
@type image: Numpy Array
@rtype: None
"""
regions = []
index = 0
y_max = self.original_image.shape[0]
x_max = self.original_image.shape[1]
for star in self.cookies:
cookie = star[0]
x = star[1]
y = star[2]
a_size = star[3]
b_size = star[4]
left_x, right_x, up_y, down_y = self.square_cookie(
a_size, b_size, x, y)
if algorithm.lower() == "hcomp":
self.h_compress[up_y:down_y, left_x:right_x] = cookie
elif algorithm.lower() == "bs":
self.compressed_image[up_y:down_y, left_x:right_x] = cookie
index += 1
if algorithm.lower() == "hcomp":
fitsio.write("hcomp_" + self.image_name, self.h_compress, \
header=self.header, clobber=True)
elif algorithm.lower() == "bs":
fitsio.write("bs_" + self.image_name, self.compressed_image, \
header=self.header, clobber=True)
def merge_partial_maps(filenames,outfile,**kwargs):
filenames = np.atleast_1d(filenames)
header = fitsio.read_header(filenames[0],ext=kwargs.get('ext',1))
nside = header['NSIDE']
data = ugali.utils.fileio.load_files(filenames,**kwargs)
pix = data['PIXEL']
ndupes = len(pix) - len(np.unique(pix))
if ndupes > 0:
msg = '%i duplicate pixels during load.'%(ndupes)
raise Exception(msg)
extname = 'DISTANCE_MODULUS'
distance = ugali.utils.fileio.load_files(filenames,ext=extname)[extname]
unique_distance = np.unique(distance)
# Check if distance moduli are the same...
if np.any(distance[:len(unique_distance)] != unique_distance):
msg = "Non-matching distance modulus:"
msg += '\n'+str(distance[:len(unique_distance)])
msg += '\n'+str(unique_distance)
raise Exception(msg)
write_partial_map(outfile,data=data,nside=nside,clobber=True)
fitsio.write(outfile,{extname:unique_distance},extname=extname)
def update_cat(incat, oucat, weight):
weights = np.load(weight)
#print(weights)
cat = ft.read(incat)
cat['WEIGHT_SYSTOT'] = 1./weights
ft.write(oucat, cat) # write
print('read %s\nwrite %s'%(incat, oucat))
def get_pulse_dist(outdir, ind, batch_size=None, ret=False, noise=True):
if ind % 5000 == 0:
print("{}".format(ind))
if batch_size is None:
batch_size = np.random.randint(1, 5, size=1)
seed = (int(time.clock() * 1000) * ind) % (4294967295) #2^32-1
np.random.seed(seed=seed)
f_0 = np.random.uniform(0., 512., size=batch_size)[..., np.newaxis,
np.newaxis] #in ms
amp = np.random.uniform(0.25, 2.5, size=batch_size)[..., np.newaxis,
np.newaxis]
width = np.random.uniform(1., 20., size=batch_size)[..., np.newaxis,
np.newaxis]
slope = np.random.uniform(-15, 15, size=batch_size)[..., np.newaxis,
np.newaxis]
t = np.arange(16)[np.newaxis, ..., np.newaxis]
f = np.arange(512)[np.newaxis, np.newaxis, ...]
f0_all = f_0 + slope * t
pulse = np.sum(amp * np.exp(-0.5 * (f - f0_all)**2 / width**2.), axis=0)
if noise:
noise_level = 0.2
pulse += noise_level * np.random.random(pulse.shape)
if outdir is not None:
fitsio.write(outdir + "img_" + str(ind) + '.fits', pulse)
scipy.misc.imsave(outdir + "pulse_" + str(ind) + '.png', pulse)
if ret:
return pulse
def imstat(dataMap, outfn='stats'):
from astropy.stats import sigma_clip
from scipy.stats import mode, scoreatpercentile
array_stats = bokutil.array_stats
fnlen = len(os.path.basename(dataMap['files'][0]))
st = np.zeros(len(dataMap['flatSequence']),
dtype=[('file', 'S%d' % fnlen), ('expTime', 'f4'),
('median', '16f4'),
('mean', '16f4'), ('mode', '16f4'), ('iqr25', '16f4'),
('iqr75', '16f4'), ('iqr10', '16f4'),
('iqr90', '16f4')])
for _i, i in enumerate(dataMap['flatSequence']):
expTime = dataMap['expTime'][i]
fn = os.path.basename(dataMap['files'][i])
fits = fitsio.FITS(dataMap['files'][i])
print '%s %4.1f ' % (fn, expTime),
st['file'][_i] = fn
st['expTime'][_i] = expTime
for j, extn in enumerate(['IM%d' % n for n in range(1, 17)]):
modeVal, pix = array_stats(fits[extn].read()[dataMap['statsPix']],
method='mode',
retArray=True)
st['mode'][_i, j] = modeVal
st['mean'][_i, j] = pix.mean()
st['median'][_i, j] = np.ma.median(pix)
st['iqr25'][_i, j] = scoreatpercentile(pix, 25)
st['iqr75'][_i, j] = scoreatpercentile(pix, 75)
st['iqr10'][_i, j] = scoreatpercentile(pix, 10)
st['iqr90'][_i, j] = scoreatpercentile(pix, 90)
print '%5d ' % (modeVal),
print
fitsio.write(outfn + '.fits', st, clobber=True)
def set_astro_refine(coadd_run, srclist):
df=desdb.files.DESFiles()
outdir=df.dir('astro_refine_fits',coadd_run=coadd_run)
if not os.path.exists(outdir):
try:
print("making dir:",outdir)
os.makedirs(outdir)
except:
pass
for s in srclist:
head_file=s['astro_refine']
fits_file=df.url('astro_refine_fits',
coadd_run=coadd_run,
expname=s['expname'],
ccd=s['ccd'])
s['wcs_file'] = fits_file
if not os.path.exists(fits_file):
print("reading:",head_file)
hdata = fitsio.read_scamp_head(head_file)
print("writing:",fits_file)
fitsio.write(fits_file, None, header=hdata, clobber=True)
def run_metacal(*, tilename, output_meds_dir, bands, seed):
"""Run metacal on a tile.
Parameters
----------
tilename : str
The DES coadd tile on which to run metacal.
output_meds_dir : str
The output DEADATA/MEDS_DIR for the simulation data products.
bands : str
The bands on which to run metacal.
seed : int
The seed for the global RNG.
"""
meds_files = [
get_meds_file_path(meds_dir=output_meds_dir,
medsconf=MEDSCONF,
tilename=tilename,
band=band) for band in bands
]
with NGMixMEDS(meds_files[0]) as m:
cat = m.get_cat()
logger.info(' meds files %s', meds_files)
n_chunks = joblib.externals.loky.cpu_count()
n_obj_per_chunk = cat.size // n_chunks
if n_obj_per_chunk * n_chunks < cat.size:
n_obj_per_chunk += 1
assert n_obj_per_chunk * n_chunks >= cat.size
logger.info(' running metacal for %d objects in %d chunks', cat.size,
n_chunks)
seeds = np.random.RandomState(seed=seed).randint(1, 2**30, size=n_chunks)
jobs = []
for chunk in range(n_chunks):
start = chunk * n_obj_per_chunk
end = min(start + n_obj_per_chunk, cat.size)
jobs.append(
joblib.delayed(_run_mcal_one_chunk)(meds_files, start, end,
seeds[chunk]))
with joblib.Parallel(n_jobs=n_chunks,
backend='loky',
verbose=50,
max_nbytes=None) as p:
outputs = p(jobs)
assert not all([o is None for o in outputs]), ("All metacal fits failed!")
output = eu.numpy_util.combine_arrlist(
[o for o in outputs if o is not None])
logger.info(' %d of %d metacal fits worked!', output.size, cat.size)
mcal_pth = get_mcal_file_path(meds_dir=output_meds_dir,
medsconf=MEDSCONF,
tilename=tilename)
logger.info(' metacal output: "%s"', mcal_pth)
make_dirs_for_file(mcal_pth)
fitsio.write(mcal_pth, output, clobber=True)
def saveClass(q,s,r,ra,dec,g,a,filename):
"""
NAME:
saveClass
PURPOSE:
save the classifications
INPUT:
q, s, r, ra, dec, g, a
filename - name of the file that the output will be saved to
OUTPUT:
(none)
HISTORY:
2011-01-30 - Written - Bovy (NYU)
"""
#Create recarray
ndata= len(q)
out= numpy.recarray((ndata,),
dtype=[('ra','f8'),
('dec','f8'),
('gamma','f8'),
('loga','f8'),
('logpx_qso','f8'),
('logpx_star','f8'),
('logpx_rrlyrae','f8')])
out.logpx_qso= q
out.logpx_star= s
out.logpx_rrlyrae= r
out.ra= ra
out.dec= dec
out.gamma= g
out.loga= a
#Now write to fits
fitsio.write(filename,out,clobber=True)
return None
def save(self, filename):
"""Save to a FITS file
:Parameters:
- `filename`: the name of the FITS file
"""
fitsio.write(filename, self.mask, header=self.header, clobber=True)
def write_fit(self):
import fitsio
npars=self['npars']
output=zeros(1, dtype=[('n','f8'),
('hlr','f8'),
('arate','f8'),
('chi2per','f8'),
('dof','f8'),
('pars','f8',npars),
('pars_norm','f8',npars),
('pars_err','f8',npars),
('pars_cov','f8',(npars,npars))])
output['n'] = self['n']
output['hlr'] = self['hlr']
output['arate'] = self.res['arate']
output['chi2per'] = self.res['chi2per']
output['dof'] = self.res['dof']
output['pars'][0,:] = self.res['pars']
output['pars_norm'][0,:] = self.res['pars_norm']
output['pars_err'][0,:] = self.res['pars_err']
output['pars_cov'][0,:,:] = self.res['pars_cov']
print(self.fits_name)
fitsio.write(self.fits_name, output, clobber=True)
def write_membership(self,filename):
"""
Write a catalog file of the likelihood region including
membership properties.
Parameters:
-----------
filename : output filename
Returns:
--------
None
"""
# Column names
name_objid = self.config['catalog']['objid_field']
name_mag_1 = self.config['catalog']['mag_1_field']
name_mag_2 = self.config['catalog']['mag_2_field']
name_mag_err_1 = self.config['catalog']['mag_err_1_field']
name_mag_err_2 = self.config['catalog']['mag_err_2_field']
# Coordinate conversion
#ra,dec = gal2cel(self.catalog.lon,self.catalog.lat)
glon,glat = self.catalog.glon_glat
ra,dec = self.catalog.ra_dec
# Angular and isochrone separations
sep = angsep(self.source.lon,self.source.lat,
self.catalog.lon,self.catalog.lat)
isosep = self.isochrone.separation(self.catalog.mag_1,self.catalog.mag_2)
# If size becomes an issue we can make everything float32
data = odict()
data[name_objid] = self.catalog.objid
data['GLON'] = glon
data['GLAT'] = glat
data['RA'] = ra
data['DEC'] = dec
data[name_mag_1] = self.catalog.mag_1
data[name_mag_err_1] = self.catalog.mag_err_1
data[name_mag_2] = self.catalog.mag_2
data[name_mag_err_2] = self.catalog.mag_err_2
data['COLOR'] = self.catalog.color
data['ANGSEP'] = sep.astype(np.float32)
data['ISOSEP'] = isosep.astype(np.float32)
data['PROB'] = self.p.astype(np.float32)
# HIERARCH allows header keywords longer than 8 characters
header = []
for param,value in self.source.params.items():
card = dict(name='HIERARCH %s'%param.upper(),
value=value.value,
comment=param)
header.append(card)
card = dict(name='HIERARCH %s'%'TS',value=self.ts(),
comment='test statistic')
header.append(card)
card = dict(name='HIERARCH %s'%'TIMESTAMP',value=time.asctime(),
comment='creation time')
header.append(card)
fitsio.write(filename,data,header=header,clobber=True)
def gen_write_fits(file_name, col_names, columns):
"""Write some columns to an output FITS file with the given column names.
:param file_name: The name of the file to write to.
:param col_names: A list of columns names for the given columns.
:param columns: A list of numpy arrays with the data to write.
"""
try:
import fitsio
data = numpy.empty(len(columns[0]),
dtype=[(name, 'f8') for name in col_names])
for (name, col) in zip(col_names, columns):
data[name] = col
fitsio.write(file_name, data, clobber=True)
except ImportError:
try:
import astropy.io.fits as pyfits
except:
import pyfits
cols = pyfits.ColDefs([
pyfits.Column(name=name, format='D', array=col)
for (name, col) in zip(col_names, columns)
])
# Depending on the version of pyfits, one of these should work:
try:
tbhdu = pyfits.BinTableHDU.from_columns(cols)
except:
tbhdu = pyfits.new_table(cols)
tbhdu.writeto(file_name, clobber=True)
def create_random_cat_finalise(label=''):
"""
This function removes duplicate randoms from the results of create_random_cat() and writes a fits file with the random catalog.
"""
import os
def unique(a):
order = np.lexsort(a.T)
a = a[order]
diff = np.diff(a, axis=0)
ui = np.ones(len(a), 'bool')
ui[1:] = (diff != 0).any(axis=1)
return a[ui],ui
a=np.vstack((np.load(label+'ra.npy'),np.load(label+'dec.npy'))).T
u,i=unique(a)
a=a[i]
ran=np.empty(len(a), dtype=[('ra','f8')]+[('dec','f8')])
ran['ra']=a[:,0].T
ran['dec']=a[:,1].T
os.remove(label+'ra.npy')
os.remove(label+'dec.npy')
fio.write(label+'random.fits.gz',ran,clobber=True)
return
def process_file(header, newfilename=None):
fac = 8
shape = int(header['NAXIS1']) / fac, int(header['NAXIS2']) / fac
header['NAXIS1'] = shape[0]
header['NAXIS2'] = shape[1]
header['CRPIX1'] = shape[0] / 2 + 0.5
header['CRPIX2'] = shape[1] / 2 + 0.5
header['CD1_1'] = float(header['CD1_1']) * fac
header['CD2_2'] = float(header['CD2_2']) * fac
header['IMTYPE'] = 'ebv'
header.pop('FILTER', None)
y, x = numpy.array(numpy.indices(shape), dtype='f8').reshape(2, -1)
x += 0.5
y += 0.5
world = wcs_tangent.pix2ang_hdr(numpy.array((x, y)),
header,
zero_offset=True)
ebv, junk = sfdmap.ebv(world[0], world[1])
ebv = ebv.reshape(shape)
print newfilename, world.max(axis=1), world.min(axis=1)
try:
os.unlink(newfilename)
except OSError:
pass
try:
os.makedirs(os.path.dirname(newfilename))
except OSError:
pass
fitsio.write(newfilename, ebv, header=header)
def footprint_area(cat,ngal=1,mask=None,nside=4096,nest=True,label=''):
import healpy as hp
import matplotlib
matplotlib.use ('agg')
import matplotlib.pyplot as plt
# plt.style.use('/home/troxel/SVA1/SVA1StyleSheet.mplstyle')
from matplotlib.colors import LogNorm
import pylab
mask=CatalogMethods.check_mask(cat.coadd,mask)
if not hasattr(cat, 'pix'):
cat.pix=CatalogMethods.radec_to_hpix(cat.ra,cat.dec,nside=nside,nest=True)
area=hp.nside2pixarea(nside)*(180./np.pi)**2
print 'pixel area (arcmin)', area*60**2
mask1=np.bincount(cat.pix[mask])>ngal
print 'footprint area (degree)', np.sum(mask1)*area
pix=np.arange(len(mask1))[mask1]
print pix
tmp=np.zeros((12*nside**2), dtype=[('hpix','int')])
tmp['hpix'][pix.astype(int)]=1
print tmp['hpix'][pix.astype(int)]
fio.write('footprint_hpix'+label+'.fits.gz',tmp,clobber=True)
tmp2=np.zeros(12*nside**2)
tmp2[tmp.astype(int)]=1
hp.cartview(tmp2,nest=True)
plt.savefig('footprint_hpix'+label+'.png')
plt.close()
return
def write_fits(postdata, bins, outfile, raref, decref, weights=None):
"""
write posterior data as fits file
"""
xmin = postdata[:,0].min() - 0.1*(postdata[:,0].max() - postdata[:,0].min())
xmax = postdata[:,0].max() + 0.1*(postdata[:,0].max() - postdata[:,0].min())
ymin = postdata[:,1].min() - 0.1*(postdata[:,1].max() - postdata[:,1].min())
ymax = postdata[:,1].max() + 0.1*(postdata[:,1].max() - postdata[:,1].min())
H, X, Y = np.histogram2d(postdata[:,0].flatten(), postdata[:,1].flatten(), \
bins=bins, range=[[xmin, xmax], [ymin, ymax]], \
weights=weights)
H = H.astype(np.float32)
dx = X[1] - X[0]
dy = Y[1] - Y[0]
filename = "%s_post.fits" % (outfile)
header = OrderedDict()
header["CTYPE1"] = "RA---TAN"
header["CTYPE2"] = "DEC--TAN"
header["CUNIT1"] = "deg"
header["CUNIT2"] = "deg"
header["CRVAL1"] = raref
header["CDELT1"] = dx
header["CRVAL2"] = decref
header["CDELT2"] = dy
header["WCSAXES"] = 2
header["CRPIX1"] = (-X[0])/dx
header["CRPIX2"] = (-Y[0])/dy
header["EQUINOX"] = 2000
fitsio.write(filename, H, header=header)
def test_load_append_table_memory_chunk_fits(self):
print('\n*** test_load_append_table_memory_chunk_fits ***\n')
data = create_test_data()
for i in range(4):
data = np.concatenate((data, data))
fitsio.write(self.fitsfile, data, clobber=True)
self.assertTrue(os.path.exists(self.fitsfile))
# memsize
self.con.drop_table(self.tablename)
command = "load_table %s --tablename %s --memsize %s --chunksize %s" % (
self.fitsfile, self.tablename, self.memsize, self.chunk * 10)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 16)
# appending
command = "append_table %s --tablename %s --memsize %s --chunksize %s" % (
self.fitsfile, self.tablename, self.memsize, self.chunk * 200)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 2 * 16)
# end
os.remove(self.fitsfile)
self.con.drop_table(self.tablename)
def _write_checkpoint(self, tm):
print >>stderr,'checkpointing at',tm,'seconds'
print >>stderr,self.checkpoint_file
fitsio.write(self.checkpoint_file,
self.data,
clobber=True)
self.checkpointed=True
def _consumeRead(self, path, cmd, header=None):
# /home/data/wincharis/H2RG-C17206-ASIC-104/UpTheRamp/20160712210126/H2RG_R01_M01_N01.fits
dirName, fileName = os.path.split(path)
cmd.diag('text="checking %s"' % (fileName))
match = re.match('^H2RG_R0*(\d+)_M0*(\d+)_N0*(\d+)\.fits', fileName)
if match is None:
cmd.warn("failed to split up filename: %s" % (file))
return
rampN, groupN, readN = [int(m) for m in match.group(1,2,3)]
cmd.diag('text="new read %d %d %d"' % (rampN, groupN, readN))
if readN == 1:
if header is not None:
cmd.diag('text="getting header"')
subaruHdr = header
else:
subaruHdr = pyfits.Header()
cards = [dict(name='IDLPATH', value=dirName)]
for c in subaruHdr.cards:
cards.append(dict(name=c.keyword, value=c.value, comment=c.comment))
phdu = fitsio.FITSHDR(cards)
fitsio.write(self.outfile, None, header=phdu, clobber=True)
cmd.diag('text="new file %s"' % (self.outfile))
inData, inHdr = fitsio.read(path, header=True)
stackFile = fitsio.FITS(self.outfile, mode='rw')
stackFile.write(inData, header=inHdr)
stackFile[-1].write_checksum()
stackFile.close()
cmd.inform('readN=%d,%d,%d,%s' % (rampN,groupN,readN,self.outfile))
def saveAtm(self, filename, clobber=False):
"""
Save the atmosphere to a fits file
parameters
----------
filename: string
output filename
clobber: bool, optional
clobber existing output file. Default is False
"""
import fitsio
hdr = fitsio.FITSHDR()
hdr['PMB'] = self.pmb
hdr['PWV'] = self.pwv
hdr['O3'] = self.o3
hdr['TAU'] = self.tau
hdr['LAMNORM'] = self.lambdaNorm
hdr['ALPHA'] = self.alpha
hdr['ZENITH'] = self.zenith
hdr['CO2MX'] = self.co2MX
hdr['ELEV'] = self.elevation
fitsio.write(filename, self.atm, header=hdr, clobber=clobber)
def _write_hpx_fits(pixlist):
"""from files that touch a pixel, write out objects in each pixel"""
pixnum, files = pixlist
# ADM only proceed if some files touch a pixel.
if len(files) > 0:
# ADM track if it's our first time through the files loop.
first = True
# ADM Read in files that touch a pixel.
for file in files:
filename = os.path.join(fitsdir, file)
objs = fitsio.read(filename)
# ADM only retain objects in the correct pixel.
pix = radec2pix(nside, objs["RA"], objs["DEC"])
if first:
done = objs[pix == pixnum]
first = False
else:
done = np.hstack([done, objs[pix == pixnum]])
# ADM construct the name of the output file.
outfilename = 'healpix-{:05d}.fits'.format(pixnum)
outfile = os.path.join(hpxdir, outfilename)
# ADM write out the file.
hdr = fitsio.FITSHDR()
hdr['HPXNSIDE'] = nside
hdr['HPXNEST'] = True
fitsio.write(outfile, done, extname='URATHPX', header=hdr)
return
def save_file(filename, data, header, format, unitdict=None):
basename = os.path.splitext(filename)[0]
if format == 'fits':
units = None
# ADM derive the units from the data columns if possible.
if unitdict is not None:
# ADM some columns from external-match files might not have
# ADM units, so pass an empty string for external columns.
units = [unitdict[col] if col in unitdict.keys() else ""
for col in data.dtype.names]
# ADM add the external match code version header dependency.
dep = [int(key.split("DEPNAM")[-1]) for key in header.keys()
if 'DEPNAM' in key]
if len(dep) == 0:
nextdep = 0
else:
nextdep = np.max(dep) + 1
header["DEPNAM{:02d}".format(nextdep)] = 'match_external'
header["DEPVER{:02d}".format(nextdep)] = git_version()
filename = basename + '.fits'
fitsio.write(filename, data, extname='MATCHED', header=header,
clobber=True, units=units)
elif format == 'hdf5':
filename = basename + '.hdf5'
import h5py
with h5py.File(filename, 'w') as ff:
dset = ff.create_dataset('MATCHED', data=data)
for key in header:
dset.attrs[key] = header[key]
else:
raise ValueError("Unknown format")
def test_load_append_table_memory_fits(self):
print('\n*** test_load_append_table_memory_fits ***\n')
data = create_test_data()
for i in range(4):
data = np.concatenate((data, data))
fitsio.write(self.fitsfile, data, clobber=True)
self.assertTrue(os.path.exists(self.fitsfile))
# memsize
self.con.drop_table(self.tablename)
self.assertTrue(self.con.load_table(
self.fitsfile, name=self.tablename, memsize=self.memsize))
cursor = self.con.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 16)
# appending
self.assertTrue(self.con.append_table(
self.fitsfile, name=self.tablename, memsize=self.memsize))
cursor = self.con.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 2 * 16)
# end
os.remove(self.fitsfile)
self.con.drop_table(self.tablename)
def main():
args = get_args()
shear_true = [args.shear, 0.00]
rng = np.random.RandomState(args.seed)
# let's just do R11 for simplicity and to speed up this example; typically
# the off diagonal terms are negligible, and R11 and R22 are usually
# consistent
dlist = []
for i in progress(args.ntrial, miniters=10):
obs = make_sim_obs(
rng=rng, noise=args.noise, shear=shear_true, show=args.show,
)
tdata = get_all_moments(obs=obs, rng=rng)
dlist.append(tdata)
data = np.hstack(dlist)
print_shear(data, shear_true)
if args.output is not None:
import fitsio
print('writing:', args.output)
fitsio.write(args.output, data, clobber=True)
def _write_gaia_fits(infile):
"""read an input name for a csv file and write it to FITS"""
outbase = os.path.basename(infile)
outfilename = "{}.fits".format(outbase.split(".")[0])
outfile = os.path.join(fitsdir, outfilename)
fitstable = ascii.read(infile, format='csv')
# ADM need to convert 5-string values to boolean.
cols = np.array(fitstable.dtype.names)
boolcols = cols[np.hstack(fitstable.dtype.descr)[1::2] == '<U5']
for col in boolcols:
fitstable[col] = fitstable[col] == 'true'
# ADM only write out the columns we need for targeting.
nobjs = len(fitstable)
done = np.zeros(nobjs, dtype=ingaiadatamodel.dtype)
for col in done.dtype.names:
if col == 'REF_CAT':
done[col] = 'G2'
else:
done[col] = fitstable[col.lower()]
fitsio.write(outfile, done, extname='GAIAFITS')
# ADM return the HEALPixels that this file touches.
pix = set(radec2pix(nside, fitstable["ra"], fitstable["dec"]))
return [pix, os.path.basename(outfile)]
def _get_gaia_matches(fnwdir):
'''wrapper on match_gaia_to_primary() given a file name'''
# ADM extract the output file name.
fn = os.path.basename(fnwdir)
outfile = '{}/{}'.format(outdir, fn.replace(".fits", ender))
# ADM read in the objects.
objs, hdr = io.read_tractor(fnwdir, header=True)
# ADM match to Gaia sources.
gaiainfo = match_gaia_to_primary(objs)
log.info(
'Done with Gaia match for {} primary objects...t = {:.1f}s'.format(
len(objs),
time() - start))
# ADM remove the GAIA_RA, GAIA_DEC columns as they aren't
# ADM in the imaging surveys data model.
gaiainfo = pop_gaia_coords(gaiainfo)
# ADM add the Gaia column information to the sweeps array.
for col in gaiainfo.dtype.names:
objs[col] = gaiainfo[col]
fitsio.write(outfile, objs, extname='SWEEP', header=hdr, clobber=True)
return True
def testTableWriteRead(self):
"""
Test a basic table write, data and a header, then reading back in to
check the values
"""
fname=tempfile.mktemp(prefix='fitsio-TableWrite-',suffix='.fits')
try:
with fitsio.FITS(fname,'rw',clobber=True) as fits:
try:
fits.write_table(self.data, header=self.keys, extname='mytable')
write_success=True
except:
write_success=False
self.assertTrue(write_success,"testing write does not raise an error")
if not write_success:
skipTest("cannot test result if write failed")
d = fits[1].read()
self.compare_rec(self.data, d, "table read/write")
h = fits[1].read_header()
self.compare_headerlist_header(self.keys, h)
# see if our convenience functions are working
fitsio.write(fname, self.data2,
extname="newext",
header={'ra':335.2,'dec':-25.2})
d = fitsio.read(fname, ext='newext')
self.compare_rec(self.data2, d, "table data2")
# now test read_column
with fitsio.FITS(fname) as fits:
for f in self.data.dtype.names:
d = fits[1].read_column(f)
self.compare_array(self.data[f], d, "table 1 single field read '%s'" % f)
for f in self.data2.dtype.names:
d = fits['newext'].read_column(f)
self.compare_array(self.data2[f], d, "table 2 single field read '%s'" % f)
# now list of columns
for cols in [['u2scalar','f4vec','Sarr'],
['f8scalar','u2arr','Sscalar']]:
d = fits[1].read(columns=cols)
for f in d.dtype.names:
self.compare_array(self.data[f][:], d[f], "test column list %s" % f)
rows = [1,3]
d = fits[1].read(columns=cols, rows=rows)
for f in d.dtype.names:
self.compare_array(self.data[f][rows], d[f], "test column list %s row subset" % f)
finally:
if os.path.exists(fname):
#pass
os.remove(fname)
def go(self):
"""
make fake catalog files for make-meds-input, and call
the code to make the meds scripts etc.
"""
tiles=numpy.unique( self.data['tilename'] )
tiles.sort()
ntile=len(tiles)
for i,tile in enumerate(tiles):
print '-'*70
print '%s/%s' % (i+1, ntile)
run = self.get_run(tile)
out=self.make_fake_tile_cat(tile)
# currently the same for all bands
for band in ['g','r','i','z','Y']:
fname=self.get_fake_tile_fname(tile, band)
print
print fname
fitsio.write(fname, out, clobber=True)
self.make_scripts(run, band, fname)
def _make_truth_catalog(self):
"""Make the truth catalog."""
# always done with first band
band = self.bands[0]
coadd_wcs = get_esutil_wcs(
image_path=self.info[band]['image_path'],
image_ext=self.info[band]['image_ext'])
ra, dec, x, y = make_coadd_grid_radec(
rng=self.gal_rng, coadd_wcs=coadd_wcs,
return_xy=True, n_grid=self.gal_kws['n_grid'])
truth_cat = np.zeros(
len(ra), dtype=[
('number', 'i8'),
('ra', 'f8'),
('dec', 'f8'),
('x', 'f8'),
('y', 'f8')])
truth_cat['number'] = np.arange(len(ra)).astype(np.int64) + 1
truth_cat['ra'] = ra
truth_cat['dec'] = dec
truth_cat['x'] = x
truth_cat['y'] = y
truth_cat_path = get_truth_catalog_path(
meds_dir=self.output_meds_dir,
medsconf=MEDSCONF,
tilename=self.tilename)
make_dirs_for_file(truth_cat_path)
fitsio.write(truth_cat_path, truth_cat, clobber=True)
return truth_cat
def match_ps1mds(matchRad=2.5):
raise NotImplementedError
pstiles = panstarrs_md_tiles(observed=True)
for field, tiles in pstiles.items():
stars = fitsio.read(ps1md_starfile(field))
matches = match_objects(stars, tiles)
fitsio.write('ps1%s_match.fits' % field, matches, clobber=True)
def makeReferenceMatchesFromFits(self, refLoader, clobber=False):
"""
Make an absolute reference match catalog, saving to fits.
Parameters
----------
refLoader: `object`
Object which has refLoader.getFgcmReferenceStarsHealpix
clobber: `bool`, optional
Clobber existing absref catalog? Default is False.
"""
import fitsio
refFile = self.starConfig['starfileBase'] + '_refcat.fits'
if not clobber:
if os.path.isfile(refFile):
self.fgcmLog.info("Found %s" % (refFile))
return refFile
self.makeReferenceMatches(refLoader)
fitsio.write(refFile, self.referenceCat, clobber=True)
return refFile
def generate_jk_centers_from_mask(outfile, regionfile, nrand=1e5):
with h5py.File(outfile, 'r') as f:
mask = f['index/mask/hpix'][:]
nside = 4096
pmap = np.zeros(12 * nside**2)
pmap[mask] = 1
pmap = hu.DensityMap('nest', pmap)
rand_ra, rand_dec = pmap.genrand(int(nrand), system='eq')
centers = KMeans(n_clusters=1000,
random_state=0).fit(np.vstack([rand_ra, rand_dec]).T)
centers = centers.cluster_centers_
kdt = spatial.cKDTree(centers)
dist, idx = kdt.query(centers, 2)
centers_dist = np.zeros((1000, 3))
centers_dist[:, :2] = centers
centers_dist[:, 2] = dist[:, 1]
fitsio.write(regionfile, centers_dist)
def create_random_cat_finalise(label=''):
"""
This function removes duplicate randoms from the results of create_random_cat() and writes a fits file with the random catalog.
"""
import os
def unique(a):
order = np.lexsort(a.T)
a = a[order]
diff = np.diff(a, axis=0)
ui = np.ones(len(a), 'bool')
ui[1:] = (diff != 0).any(axis=1)
return a[ui], ui
a = np.vstack(
(np.load(label + 'ra.npy'), np.load(label + 'dec.npy'))).T
u, i = unique(a)
a = a[i]
ran = np.empty(len(a), dtype=[('ra', 'f8')] + [('dec', 'f8')])
ran['ra'] = a[:, 0].T
ran['dec'] = a[:, 1].T
os.remove(label + 'ra.npy')
os.remove(label + 'dec.npy')
fio.write(label + 'random.fits.gz', ran, clobber=True)
return
def _write_urat_fits(infile):
"""read an input name for a csv file and write it to FITS"""
outbase = os.path.basename(infile)
outfilename = "{}.fits".format(outbase.split(".")[0])
outfile = os.path.join(fitsdir, outfilename)
# ADM astropy understands without specifying format='csv'.
fitstable = ascii.read(infile)
# ADM map the ascii-read csv to typical DESI quantities.
nobjs = len(fitstable)
done = np.zeros(nobjs, dtype=uratdatamodel.dtype)
# ADM have to do this one-by-one, given the format.
done["RA"] = fitstable['col1'] / 1000. / 3600.
done["DEC"] = fitstable['col2'] / 1000. / 3600. - 90.
done["PMRA"] = fitstable['col16'] / 10.
done["PMDEC"] = fitstable['col17'] / 10.
done["PM_ERROR"] = fitstable['col18'] / 10.
done["APASS_G_MAG"] = fitstable['col36'] / 1000.
done["APASS_R_MAG"] = fitstable['col37'] / 1000.
done["APASS_I_MAG"] = fitstable['col38'] / 1000.
done["APASS_G_MAG_ERROR"] = fitstable['col41'] / 1000.
done["APASS_R_MAG_ERROR"] = fitstable['col42'] / 1000.
done["APASS_I_MAG_ERROR"] = fitstable['col43'] / 1000.
done["URAT_ID"] = fitstable['col46']
fitsio.write(outfile, done, extname='URATFITS')
# ADM return the HEALPixels that this file touches.
pix = set(radec2pix(nside, done["RA"], done["DEC"]))
return [pix, os.path.basename(outfile)]
def combine_ccds(ccds, output):
""" Combines CCD annotated files
"""
columns = ['camera', 'filter', 'fwhm', 'mjd_obs', 'exptime',
'ra', 'dec', 'ra0','ra1','ra2','ra3','dec0','dec1','dec2','dec3',
'galdepth', 'ebv', 'airmass', 'ccdskycounts', 'pixscale_mean', 'ccdzpt']
dtype = np.dtype([('camera', '<U7'),('filter', '<U1'), ('exptime', '>f4'), ('mjd_obs', '>f8'),
('airmass', '>f4'), ('fwhm', '>f4'), ('ra', '>f8'), ('dec', '>f8'), ('ccdzpt', '>f4'),
('ccdskycounts', '>f4'), ('ra0', '>f8'), ('dec0', '>f8'), ('ra1', '>f8'),
('dec1', '>f8'), ('ra2', '>f8'), ('dec2', '>f8'), ('ra3', '>f8'), ('dec3', '>f8'),
('pixscale_mean', '>f4'), ('ebv', '>f4'), ('galdepth', '>f4')])
# read each ccd file > fix its dtype > move on to the next
ccds_data = []
for ccd_i in ccds:
print('working on .... %s'%ccd_i.split('/')[-1])
data_in = ft.FITS(ccd_i)[1].read(columns=columns)
data_out = fixdtype(data_in, dtype)
in_diff = np.setdiff1d(dtype.descr, data_in.dtype.descr)
out_diff = np.setdiff1d(dtype.descr, data_out.dtype.descr)
print(f'number of ccds in this file: {data_in.size}')
print(f'different dtypes (before): {in_diff}')
print(f'different dtypes (after): {out_diff}')
ccds_data.append(data_out)
ccds_data_c = np.concatenate(ccds_data)
print(f'Total number of combined ccds : {ccds_data_c.size}')
ft.write(output, ccds_data_c, clobber=True)
print(f'wrote the combined ccd file: {output}')
def write_astrometry_for_gfa_file(fn,
out_dir,
gnums=[0, 2, 3, 5, 7, 8],
left=False,
config_fn=None):
hdr = fitsio.read_header(fn, ext=1)
skyra = hdr['SKYRA']
skydec = hdr['SKYDEC']
expnum = hdr['EXPID']
imgfns = []
crpix_arg = '--crpix-center'
for gnum in gnums:
gname = 'GUIDE%i' % gnum
g, x0, y0 = read_guide_image_file(fn, ext=gname)
good = read_good_pix_maps(gname)
imgfn = os.path.join(out_dir, 'gfa-%i-%s.fits' % (expnum, gname))
g = g * good
gh, gw = g.shape
if left:
g = g[:, :gw // 2]
crpix_arg = '--crpix-x %.1f --crpix-y %.1f' % (gw + 0.5,
gh / 2 + 0.5)
fitsio.write(imgfn, g, clobber=True)
imgfns.append(imgfn)
if out_dir == '':
out_dir = '.'
if config_fn is None:
config_fn = commish_paths.an_config_filename
# NOTE -- definitely want --tweak-order 1, otherwise (with
# --tweak-order 0 or --no-tweak) we get a SQUARE CD matrix!!
cmd = ''
an_dir = commish_paths.an_path
if an_dir is not None:
cmd = ('PATH=%s/bin:${PATH} ' % an_dir) + cmd
an_py_path = commish_paths.an_py_path
if an_py_path is not None:
cmd = ('PYTHONPATH=%s:${PYTHONPATH} ' % an_py_path) + cmd
cmd += ((
'solve-field --config %s --xscale 1.1' +
' --ra %f --dec %f --radius 2 --scale-low 0.18 --scale-high 0.24 --scale-units app --downsample 2'
+ ' --continue --tweak-order 1 --plot-scale 0.5 --objs 100' +
' --batch --dir %s %s ') %
(config_fn, skyra, skydec, out_dir, crpix_arg))
cmd += ' '.join(imgfns)
print(cmd)
os.system(cmd)
wcsfns = [fn.replace('.fits', '.wcs') for fn in imgfns]
wcsfns = [fn if os.path.exists(fn) else None for fn in wcsfns]
return wcsfns, hdr
def test_load_append_table_memory_chunk_fits(self):
print('\n*** test_load_append_table_memory_chunk_fits ***\n')
data = create_test_data()
for i in range(4):
data = np.concatenate((data, data))
fitsio.write(self.fitsfile, data, clobber=True)
self.assertTrue(os.path.exists(self.fitsfile))
# memsize
self.con.drop_table(self.tablename)
command = "load_table %s --tablename %s --memsize %s --chunksize %s" % (
self.fitsfile, self.tablename, self.memsize, self.chunk * 10)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 16)
# appending
command = "append_table %s --tablename %s --memsize %s --chunksize %s" % (
self.fitsfile, self.tablename, self.memsize, self.chunk * 200)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 2 * 16)
# end
os.remove(self.fitsfile)
self.con.drop_table(self.tablename)
def write_rgb():
#g,r,z = [fitsio.read('detmap-%s.fits' % band) for band in 'grz']
g,r,z = [fitsio.read('coadd-%s.fits' % band) for band in 'grz']
plt.figure(figsize=(10,10))
plt.subplots_adjust(left=0.05, right=0.95, bottom=0.05, top=0.95)
plt.clf()
for (im1,cc),scale in zip([(g,'b'),(r,'g'),(z,'r')],
[2.0, 1.2, 0.4]):
im = im1 * scale
im = im[im != 0]
plt.hist(im.ravel(), histtype='step', color=cc,
range=[np.percentile(im, p) for p in (1,98)], bins=50)
ps.savefig()
#rgb = get_rgb_image(g,r,z, alpha=0.8, m=0.02)
#rgb = get_rgb_image(g,r,z, alpha=16., m=0.005, m2=0.002,
#rgb = get_rgb_image(g,r,z, alpha=32., m=0.01, m2=0.002,
rgb = get_rgb_image(g,r,z, alpha=8., m=0.0, m2=0.0,
scale_g = 2.,
scale_r = 1.1,
scale_z = 0.5,
Q = 10)
#for im in g,r,z:
# mn,mx = [np.percentile(im, p) for p in [20,99]]
# print 'mn,mx:', mn,mx
plt.clf()
plt.imshow(rgb, interpolation='nearest', origin='lower')
ps.savefig()
fitsio.write('rgb.fits', rgb)
def plotMassFunction(im, pm, outbase, mmin=9, mmax=13, mstep=0.05):
"""
Make a comparison plot between the input mass function and the
predicted projected correlation function
"""
plt.clf()
nmbins = ( mmax - mmin ) / mstep
mbins = np.logspace( mmin, mmax, nmbins )
mcen = ( mbins[:-1] + mbins[1:] ) /2
plt.xscale( 'log', nonposx = 'clip' )
plt.yscale( 'log', nonposy = 'clip' )
ic, e, p = plt.hist( im, mbins, label='Original Halos', alpha=0.5, normed = True)
pc, e, p = plt.hist( pm, mbins, label='Added Halos', alpha=0.5, normed = True)
plt.legend()
plt.xlabel( r'$M_{vir}$' )
plt.ylabel( r'$\frac{dN}{dM}$' )
#plt.tight_layout()
plt.savefig( outbase+'_mfcn.png' )
mdtype = np.dtype( [ ('mcen', float), ('imcounts', float), ('pmcounts', float) ] )
mf = np.ndarray( len(mcen), dtype = mdtype )
mf[ 'mcen' ] = mcen
mf[ 'imcounts' ] = ic
mf[ 'pmcounts' ] = pc
fitsio.write( outbase+'_mfcn.fit', mf )
def test_load_append_table_memory_fits(self):
print('\n*** test_load_append_table_memory_fits ***\n')
data = create_test_data()
for i in range(4):
data = np.concatenate((data, data))
fitsio.write(self.fitsfile, data, clobber=True)
self.assertTrue(os.path.exists(self.fitsfile))
# memsize
self.con.drop_table(self.tablename)
self.assertTrue(
self.con.load_table(self.fitsfile,
name=self.tablename,
memsize=self.memsize))
cursor = self.con.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 16)
# appending
self.assertTrue(
self.con.append_table(self.fitsfile,
name=self.tablename,
memsize=self.memsize))
cursor = self.con.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 2 * 16)
## end
os.remove(self.fitsfile)
self.con.drop_table(self.tablename)
def plotFeaturePDF(ift, pft, outbase, fmin=0.0, fmax=1.0, fstep=0.01):
"""
Plot a comparison between the input feature distribution and the
feature distribution of the predicted halos
"""
plt.clf()
nfbins = ( fmax - fmin ) / fstep
fbins = np.logspace( fmin, fmax, nfbins )
fcen = ( fbins[:-1] + fbins[1:] ) / 2
plt.xscale( 'log', nonposx='clip' )
plt.yscale( 'log', nonposy='clip' )
ic, e, p = plt.hist( ift, fbins, label='Original Halos', alpha=0.5, normed=True )
pc, e, p = plt.hist( pft, fbins, label='Added Halos', alpha=0.5, normed=True )
plt.legend()
plt.xlabel( r'$\delta$' )
plt.savefig( outbase+'_fpdf.png' )
fdtype = np.dtype( [ ('fcen', float), ('ifcounts', float), ('pfcounts', float) ] )
fd = np.ndarray( len(fcen), dtype = fdtype )
fd[ 'mcen' ] = fcen
fd[ 'imcounts' ] = ic
fd[ 'pmcounts' ] = pc
fitsio.write( outbase+'_fpdf.fit', fd )
def test_load_append_table_fits(self):
print('\n*** test_load_append_table_fits ***\n')
data = create_test_data()
fitsio.write(self.fitsfile, data, clobber=True)
self.assertTrue(os.path.exists(self.fitsfile))
self.con.drop_table(os.path.splitext(self.fitsfile)[0].upper())
# name from filename
command = "load_table %s " % self.fitsfile
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' %
os.path.splitext(self.fitsfile)[0].upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows)
# appending
command = "append_table %s " % self.fitsfile
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' %
os.path.splitext(self.fitsfile)[0].upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows * 2)
self.con.drop_table(os.path.splitext(self.fitsfile)[0].upper())
os.remove(self.fitsfile)
def create_test_fits(filename=None, data=None):
if filename is None:
filename = BASENAME + '.fits'
if data is None:
data = create_test_data()
fitsio.write(filename, data)
return filename
def gen_write_fits(file_name, col_names, columns):
"""Write some columns to an output FITS file with the given column names.
:param file_name: The name of the file to write to.
:param col_names: A list of columns names for the given columns.
:param columns: A list of numpy arrays with the data to write.
"""
import numpy
try:
import fitsio
data = numpy.empty(len(columns[0]), dtype=[ (name,'f8') for name in col_names ])
for (name, col) in zip(col_names, columns):
data[name] = col
fitsio.write(file_name, data, clobber=True)
except ImportError:
try:
import astropy.io.fits as pyfits
except:
import pyfits
cols = pyfits.ColDefs([
pyfits.Column(name=name, format='D', array=col)
for (name, col) in zip(col_names, columns) ])
# Depending on the version of pyfits, one of these should work:
try:
tbhdu = pyfits.BinTableHDU.from_columns(cols)
except:
tbhdu = pyfits.new_table(cols)
tbhdu.writeto(file_name, clobber=True)
def _partition_gaia(filename=None, dr='dr1'):
"""Partition a Gaia source FITS files
Parameters
----------
filename: string
base name of partition file
dr : string
name of data release (default 'dr1')
Comments
--------
Breaks file into RA and Dec bins. Creates or adds to Gaia partition files..
"""
gaia = fitsio.read(filename, ext=1)
ira = np.int32(gaia['ra'])
idec = np.int32((gaia['dec'] + 90.))
isp = np.nonzero(idec == 0)[0]
ira[isp] = 0
inp = np.nonzero(idec == 179)[0]
ira[inp] = 0
ifile = idec * 360 + ira
uniqs, iuniqs = np.unique(ifile, return_index=True)
for iuniq in iuniqs:
cra = ira[iuniq]
cdec = idec[iuniq]
cgaia = np.nonzero((ira == cra) & (idec == cdec))[0]
filename = _partition_filename(cra, cdec, dr=dr)
if (os.path.isfile(filename)):
fits = fitsio.FITS(filename, 'rw')
fits[-1].append(gaia[cgaia])
fits.close()
else:
fitsio.write(filename, gaia[cgaia])
return
def select_ndwfs_stars():
ndwfsdir = '/global/scratch2/sd/imcgreer/ndwfs/DR3/matchedFITS/'
dtype = [('number', 'i4'), ('autoMag', '3f4'), ('autoMagErr', '3f4'),
('ra', 'f8'), ('dec', 'f8'), ('rFWHM', 'f4'), ('rClass', 'f4')]
starcat = []
rcols = [
'NUMBER', 'MAG_AUTO', 'MAGERR_AUTO', 'ALPHA_J2000', 'DELTA_J2000',
'FWHM_IMAGE', 'CLASS_STAR'
]
cols = ['MAG_AUTO', 'MAGERR_AUTO']
for dec1 in range(32, 36):
catfn = lambda b: 'NDWFS_%s_%d_%d_cat_m.fits.gz' % (b, dec1, dec1 + 1)
rfits = fitsio.FITS(ndwfsdir + catfn('R'))
bfits = fitsio.FITS(ndwfsdir + catfn('Bw'))
ifits = fitsio.FITS(ndwfsdir + catfn('I'))
w = rfits[1].where('FWHM_IMAGE < 7 && MAG_AUTO < 24.0 && FLAGS == 0')
print len(w)
rcat = rfits[1].read(rows=w, columns=rcols)
bcat = bfits[1].read(rows=w, columns=cols)
icat = ifits[1].read(rows=w, columns=cols)
stars = np.empty(len(w), dtype=dtype)
stars['number'] = rcat['NUMBER']
stars['ra'] = rcat['ALPHA_J2000']
stars['dec'] = rcat['DELTA_J2000']
stars['rFWHM'] = rcat['FWHM_IMAGE']
stars['rClass'] = rcat['CLASS_STAR']
for j, cat in enumerate([bcat, rcat, icat]):
stars['autoMag'][:, j] = cat['MAG_AUTO']
stars['autoMagErr'][:, j] = cat['MAGERR_AUTO']
starcat.append(stars)
starcat = np.concatenate(starcat)
fitsio.write(ndwfs_starfile, starcat, clobber=True)
def _select_stars(self):
import fitsio
print("selecting stars")
ntile=len(self.alldata)
dt=[('tilename','S30'),
('ra','f8'),
('dec','f8'),
('nstars','i4')]
stardata=numpy.zeros(ntile,dtype=dt)
for i,dd in enumerate(self.alldata):
cat=dd['data']
w,=numpy.where( (cat['flags']==0)
& (numpy.abs(cat['spread_model']) < 0.002)
& (cat['mag_auto'] < 19) & (cat['mag_auto'] > 16) )
print(" ",dd['coadd_run'])
print(" nstars:",w.size)
dd['wstars'] = w
stardata['tilename'][i] = dd['tilename']
stardata['ra'][i] = numpy.median(cat['alphawin_j2000'][w])
stardata['dec'][i] = numpy.median(cat['deltawin_j2000'][w])
stardata['nstars'][i] = w.size
self.stardata=stardata
sdfile=get_stardata_file(self.release)
print(sdfile)
fitsio.write(sdfile,self.stardata,clobber=True)
def save_weights(cat, val, zbin, w, bins, mask0):
for i in xrange(cat.sbins):
if cat.cat == 'mcal':
mask = bins[i][np.in1d(bins[i],
np.where(w != 0)[0],
assume_unique=False)]
else:
mask = (bins == i) & mask0 & (w != 0)
pix = catalog.CatalogMethods.radec_to_hpix(cat.ra[mask],
cat.dec[mask],
nside=4096,
nest=False)
upix = np.unique(pix)
w0 = w[mask]
w1 = np.bincount(pix, weights=w0)
w2 = np.bincount(pix, weights=w0 * w0)
mask = np.where(w1 != 0)[0]
# upix = upix[mask]
w1 = w1[mask] / np.bincount(pix)[mask]
w2 = w2[mask] / np.bincount(pix)[mask]
out = np.empty(len(upix),
dtype=[('pix', int)] + [('weight', 'f4')] +
[('weightsq', 'f4')])
out['pix'] = upix
out['weight'] = w1
out['weightsq'] = w2
fio.write('text/pzrw_' + cat.name + '_' + val + '_' +
str(zbin + 1) + '_' + str(i) + '.fits.gz',
out,
clobber=True)
return
def save(self, filename):
"""Save to a FITS file
:Parameters:
- `filename`: the name of the FITS file
"""
fitsio.write(filename, self.mask, header=self.header,
clobber=True)
def write(self, fname):
"""
write to output file
"""
files.makedir_fromfile(fname)
data=self.get_data()
print("writing:",fname)
fitsio.write(fname, self.data, clobber=True)
def build_special_points_fits():
"""
Combines parts of special points catalog into single fits catalog.
"""
import fitsio as fio
tmp=fio.FITS(config.wcsfile)[-1].read()
a=np.sort(np.unique(tmp['expnum']))
b=np.sort(np.unique(tmp['ccdnum']))-1
store=np.empty((len(a)*(len(b)+1)),dtype=[('exposure',int)]+[('ccd',int)]+[('type',int)]+[('ra','f8')]+[('dec','f8')])
for i in range(len(a)):
store['exposure'][i*len(b):(i+1)*len(b)]=a[i]
for j in range(len(b)):
store['ccd'][i*len(b)+j]=b[j]
for i in range(40):
print i
tmp=np.genfromtxt('y1a1_special_points_'+str(i)+'.txt',names=['index','exposure','ccd','racenter','deccenter','rall','decll','raul','decul','ralr','declr','raur','decur'])
mask1=(store['exposure']==tmp['exposure'][j])
for j in range(len(tmp)):
if j%1000==0:
print j
mask=mask1&(store['ccd']==tmp['ccd'][j])
if tmp['racenter'][j]!=999:
store['type'][mask]=0
store['ra'][mask]=tmp['racenter'][j]
store['dec'][mask]=tmp['deccenter'][j]
if tmp['rall'][j]!=999:
store['type'][mask]=1
store['ra'][mask]=tmp['rall'][j]
store['dec'][mask]=tmp['decll'][j]
if tmp['raul'][j]!=999:
store['type'][mask]=2
store['ra'][mask]=tmp['raul'][j]
store['dec'][mask]=tmp['decul'][j]
if tmp['ralr'][j]!=999:
store['type'][mask]=3
store['ra'][mask]=tmp['ralr'][j]
store['dec'][mask]=tmp['declr'][j]
if tmp['raur'][j]!=999:
store['type'][mask]=4
store['ra'][mask]=tmp['raur'][j]
store['dec'][mask]=tmp['decur'][j]
for i in range(len(a)):
if i%1000==0:
print i
store['exposure'][len(a)*len(b)+i]=a[i]
store['ccd'][len(a)*len(b)+i]=-1
store['type'][len(a)*len(b)+i]=-1
mask=(store['exposure']==store['exposure'][len(a)*len(b)+i])&(store['type']==0)&((store['ccd']==27)|(store['ccd']==34))
store['ra'][len(a)*len(b)+i]=np.mean(store['ra'][mask])
store['dec'][len(a)*len(b)+i]=np.mean(store['dec'][mask])
fio.write(config.spointsfile,store,clobber=True)
return
def save_cat(cat):
for x in dir(cat):
obj = getattr(cat,x)
if isinstance(obj,np.ndarray):
if len(obj)==len(cat.coadd):
fio.write(x+'.fits.gz',obj,clobber=True)
return
def calc_branch_sky_noise(**keys):
"""
Calculate the sky noise in all images from the specified branch
"""
import fitsio
nsub=files.get_nsub(**keys)
deep=keys.get('deep',False)
if deep:
outfile=files.get_deep_skynoise_file(**keys)
fmin=-0.08
fmax= 0.035
binsize=0.001
else:
outfile=files.get_skynoise_file(**keys)
fmin=-0.3
fmax= 0.15
binsize=0.005
print("will write to:",outfile)
d=files.get_skynoise_plot_dir(**keys)
if not os.path.exists(d):
os.makedirs(d)
out=numpy.zeros(1,dtype=[('subid','f8'),
('skysig','f8'),
('skysig_err','f8')])
if deep:
im=files.read_deep_gal_image(**keys)
plot_file=files.get_deep_skynoise_plot_file(**keys)
else:
im=files.read_gal_image(**keys)
plot_file=files.get_skynoise_plot_file(**keys)
gf = get_sky_noise(im,fmin,fmax,binsize)
res=gf.get_result()
plt=gf.make_plot(show=False)
print(" writing:",plot_file)
plt.write_eps(plot_file)
pars=res['pars']
perr=res['perr']
print(' %.3g +/- %.3g' % (pars[1],perr[1]))
out['subid']=keys['subid']
out['skysig']=pars[1]
out['skysig_err']=perr[1]
print("writing:",outfile)
fitsio.write(outfile, out, clobber=True)
def writeFits(self,outPath,im_data,header,objName,band,im_type,first):
if not first: fits2 = fitsio.FITS(outPath,'rw')
header['BAND'] = band
header['OBJECT'] = objName
header['TYPE'] = im_type
if first:
fitsio.write(outPath,im_data,header=header)
else:
fits2.write(im_data,header=header)
def write_membership(loglike,filename):
"""
Write a catalog file of the likelihood region including
membership properties.
Parameters:
-----------
loglike : input loglikelihood object
filename : output filename
Returns:
--------
None
"""
ra,dec = gal2cel(loglike.catalog.lon,loglike.catalog.lat)
name_objid = loglike.config['catalog']['objid_field']
name_mag_1 = loglike.config['catalog']['mag_1_field']
name_mag_2 = loglike.config['catalog']['mag_2_field']
name_mag_err_1 = loglike.config['catalog']['mag_err_1_field']
name_mag_err_2 = loglike.config['catalog']['mag_err_2_field']
# Angular and isochrone separations
sep = angsep(loglike.source.lon,loglike.source.lat,
loglike.catalog.lon,loglike.catalog.lat)
isosep = loglike.isochrone.separation(loglike.catalog.mag_1,loglike.catalog.mag_2)
data = odict()
data[name_objid] = loglike.catalog.objid
data['GLON'] = loglike.catalog.lon
data['GLAT'] = loglike.catalog.lat
data['RA'] = ra
data['DEC'] = dec
data[name_mag_1] = loglike.catalog.mag_1
data[name_mag_err_1] = loglike.catalog.mag_err_1
data[name_mag_2] = loglike.catalog.mag_2
data[name_mag_err_2] = loglike.catalog.mag_err_2
data['COLOR'] = loglike.catalog.color
data['ANGSEP'] = sep
data['ISOSEP'] = isosep
data['PROB'] = loglike.p
# HIERARCH allows header keywords longer than 8 characters
header = []
for param,value in loglike.source.params.items():
card = dict(name='HIERARCH %s'%param.upper(),
value=value.value,
comment=param)
header.append(card)
card = dict(name='HIERARCH %s'%'TS',value=loglike.ts(),
comment='test statistic')
header.append(card)
card = dict(name='HIERARCH %s'%'TIMESTAMP',value=time.asctime(),
comment='creation time')
header.append(card)
fitsio.write(filename,data,header=header,clobber=True)
def write(self, outfile):
"""
Save the likelihood results as a sparse HEALPix map.
"""
data = odict()
data['PIXEL']=self.roi.pixels_target
# Full data output (too large for survey)
if self.config['scan']['full_pdf']:
data['LOG_LIKELIHOOD']=self.log_likelihood_sparse_array.T
data['RICHNESS']=self.richness_sparse_array.T
data['RICHNESS_LOWER']=self.richness_lower_sparse_array.T
data['RICHNESS_UPPER']=self.richness_upper_sparse_array.T
data['RICHNESS_LIMIT']=self.richness_upper_limit_sparse_array.T
#data['STELLAR_MASS']=self.stellar_mass_sparse_array.T
data['FRACTION_OBSERVABLE']=self.fraction_observable_sparse_array.T
else:
data['LOG_LIKELIHOOD']=self.log_likelihood_sparse_array.T
data['RICHNESS']=self.richness_sparse_array.T
data['FRACTION_OBSERVABLE']=self.fraction_observable_sparse_array.T
# Convert to 32bit float
for k in list(data.keys())[1:]:
data[k] = data[k].astype('f4',copy=False)
# Stellar mass can be calculated from STELLAR * RICHNESS
header = odict()
header['STELLAR']=round(self.stellar_mass_conversion,8)
header['LKDNSIDE']=self.config['coords']['nside_likelihood']
header['LKDPIX']=ang2pix(self.config['coords']['nside_likelihood'],
self.roi.lon,self.roi.lat)
header['NROI']=self.roi.inROI(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NANNULUS']=self.roi.inAnnulus(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NINSIDE']=self.roi.inInterior(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
header['NTARGET']=self.roi.inTarget(self.loglike.catalog_roi.lon,
self.loglike.catalog_roi.lat).sum()
# Flatten if there is only a single distance modulus
# ADW: Is this really what we want to do?
if len(self.distance_modulus_array) == 1:
for key in data:
data[key] = data[key].flatten()
logger.info("Writing %s..."%outfile)
write_partial_map(outfile,data,
nside=self.config['coords']['nside_pixel'],
header=header,
clobber=True
)
fitsio.write(outfile,
dict(DISTANCE_MODULUS=self.distance_modulus_array.astype('f4',copy=False)),
extname='DISTANCE_MODULUS',
clobber=False)
def inject(kicid, rng=6):
# Download the data.
client = kplr.API()
kic = client.star(kicid)
lcs = kic.get_light_curves(short_cadence=False)
lc = lcs[np.random.randint(len(lcs))]
# Read the data.
data = lc.read()
t = data["TIME"]
f = data["SAP_FLUX"]
fe = data["SAP_FLUX_ERR"]
q = data["SAP_QUALITY"]
# Remove missing points.
m = np.isfinite(t) * np.isfinite(f) * np.isfinite(fe) * (q == 0)
t, f, fe = t[m], f[m], fe[m]
t -= t.min()
# Build the transit system.
s = transit.System(transit.Central(q1=np.random.rand(),
q2=np.random.rand()))
body = transit.Body(period=365.25, b=np.random.rand(), r=0.04,
t0=np.random.uniform(t.max()))
s.add_body(body)
# Compute the transit model.
texp = kplr.EXPOSURE_TIMES[1] / 86400.0 # Long cadence exposure time.
model = s.light_curve(t, texp=texp)
f *= model
# Trim the dataset to include data only near the transit.
m = np.abs(t - body.t0) < rng
t, f, fe = t[m], f[m], fe[m]
t -= body.t0
# Save the injection as a FITS light curve.
dt = [("TIME", float), ("SAP_FLUX", float), ("SAP_FLUX_ERR", float)]
data = np.array(zip(t, f, fe), dtype=dt)
hdr = dict(b=body.b, period=body.period, r=body.r, t0=0.0,
q1=s.central.q1, q2=s.central.q2)
fitsio.write("{0}-injection.fits".format(kicid), data, header=hdr,
clobber=True)
# Plot the light curve.
ppm = (f / np.median(f) - 1) * 1e6
fig = pl.figure(figsize=(6, 6))
ax = fig.add_subplot(111)
ax.plot(t, ppm, ".k")
ax.set_xlim(-rng, rng)
ax.set_xlabel("time since transit [days]")
ax.set_ylabel("relative flux [ppm]")
ax.set_title("raw light curve")
fig.subplots_adjust(left=0.2, bottom=0.2, top=0.9, right=0.9)
fig.savefig("{0}-raw.pdf".format(kicid))
def test_load_table_fits(self):
data = create_test_data()
fitsio.write(self.fitsfile, data, clobber=True)
self.assertTrue(os.path.exists(self.fitsfile))
self.con.drop_table(os.path.splitext(self.fitsfile)[0].upper())
# name from filename
command = "load_table %s " % self.fitsfile
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.fitsfile)[0].upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows)
# appending
command = "append_table %s " % self.fitsfile
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % os.path.splitext(self.fitsfile)[0].upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows*2)
self.con.drop_table(os.path.splitext(self.fitsfile)[0].upper())
# name from tablename
self.con.drop_table(self.tablename)
command = "load_table %s --tablename %s" % (self.fitsfile, self.tablename)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows)
# appending
command = "append_table %s --tablename %s" % (self.fitsfile, self.tablename)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows*2)
# chunksize
self.con.drop_table(self.tablename)
command = "load_table %s --tablename %s --chunksize %s" % (self.fitsfile, self.tablename, self.chunk)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows)
## appending
command = "append_table %s --tablename %s --chunksize %s" % (self.fitsfile, self.tablename, self.chunk)
self.con.onecmd(command)
cursor = self.con2.cursor()
temp = cursor.execute('select RA,DEC from %s' % self.tablename.upper())
fetch = temp.fetchall()
self.assertEqual(len(fetch), self.nrows*2)
os.remove(self.fitsfile)
self.con.drop_table(self.tablename)
def writeCandidates(self,filename=None):
if filename is None: filename = self.candfile
threshold = self.config['search']['cand_threshold']
select = (self.assocs['CUT']==0)
select &= (self.assocs['TS']>threshold)
#select &= (self.assocs['ASSOC2']=='')
self.candidates = self.assocs[select]
logger.info("Writing %s..."%filename)
fitsio.write(filename,self.candidates,clobber=True)
