def test_update_UserCommands():
import simmetis as sim
cmd = sim.UserCommands()
cmd.update({'OBS_EXPTIME': 30})
assert cmd.cmds['OBS_EXPTIME'] == 30
with pytest.raises(KeyError):
cmd.update({'NO_EXISTE': 30})
def calculate_zeropoint(filter_name,verbose=False):
cmd = sim.UserCommands("../notebooks/metis_image_NQ.config")
cmd["ATMO_USE_ATMO_BG"] = "yes"
cmd["SCOPE_USE_MIRROR_BG"] = "yes"
cmd["SIM_VERBOSE"]="no"
cmd["FPA_QE"]="../data/TC_detector_METIS_NQ_no_losses.dat"
cmd["INST_FILTER_TC"]="../data/TC_filter_"+filter_name+".dat"
opt = sim.OpticalTrain(cmd)
fpa = sim.Detector(cmd, small_fov=False)
## generate a source with 0 mag
lam, spec = sim.source.flat_spectrum(0, "../data/TC_filter_"+filter_name+".dat")
src = sim.Source(lam=lam, spectra=np.array([spec]), ref=[0], x=[0], y=[0])
src.apply_optical_train(opt, fpa)
exptime=1
##
## noise-free image before applying Poisson noise
photonflux = fpa.chips[0].array.T
clean_image = photonflux * exptime
##
## detector image with Poisson noise
hdu = fpa.read_out(OBS_EXPTIME=exptime)
bg_counts = np.min(clean_image)
source_minus_bg_counts = np.sum(clean_image - np.min(clean_image))
if verbose:
print("Background counts/s: {0:.2E}".format(bg_counts))
print("Background-subtracted source counts/s: {0:.2E}".format(source_minus_bg_counts))
return(bg_counts,source_minus_bg_counts)
def calculate_zeropoint(filter_id, filter_path, verbose=False):
##
## find out if we have the LM or NQ band camera
filter_data = ascii.read(filter_path)
if filter_data["col1"][0] < 6:
cmd = sim.UserCommands("../notebooks/metis_image_LM.config")
cmd["FPA_QE"] = "../data/TC_detector_METIS_LM_no_losses.dat"
else:
cmd = sim.UserCommands("../notebooks/metis_image_NQ.config")
cmd["FPA_QE"] = "TC_detector_METIS_NQ_no_losses.dat"
cmd["INST_FILTER_TC"] = "../data/TC_filter_" + filter_id + ".dat"
opt = sim.OpticalTrain(cmd)
fpa = sim.Detector(cmd, small_fov=False)
## generate a source with 0 mag
lam, spec = sim.source.flat_spectrum(0, filter_path)
src = sim.Source(lam=lam, spectra=np.array([spec]), ref=[0], x=[0], y=[0])
src.apply_optical_train(opt, fpa)
exptime = 1
##
## noise-free image before applying Poisson noise
photonflux = fpa.chips[0].array.T
clean_image = photonflux * exptime
##
## detector image with Poisson noise
hdu = fpa.read_out(OBS_EXPTIME=exptime)
bg_counts = np.min(clean_image)
source_minus_bg_counts = np.sum(clean_image - np.min(clean_image))
if verbose:
print("Background counts/s: {0:.2E}".format(bg_counts))
print("Background-subtracted source counts/s: {0:.2E}".format(
source_minus_bg_counts))
return (bg_counts, source_minus_bg_counts)
def run_simmetis():
import simmetis as sim
import os
cmd = sim.UserCommands()
cmd["OBS_EXPTIME"] = 3600
cmd["OBS_NDIT"] = 1
cmd["INST_FILTER_TC"] = "J"
src = sim.source.source_1E4_Msun_cluster()
opt = sim.OpticalTrain(cmd)
fpa = sim.Detector(cmd)
src.apply_optical_train(opt, fpa)
fpa.read_out("my_output.fits")
is_there = os.path.exists("my_output.fits")
os.remove("my_output.fits")
return is_there
def background_increases_consistently_with_exptime():
"""
Run an empty source for exposure time: (1,2,4,8,16,32,64) mins
If true the background level increases linearly and the stdev increases as sqrt(exptime)
"""
import numpy as np
import simmetis as sim
cmd = sim.UserCommands()
cmd["OBS_REMOVE_CONST_BG"] = "no"
opt = sim.OpticalTrain(cmd)
fpa = sim.Detector(cmd)
stats = []
for t in [2**i for i in range(7)]:
src = sim.source.empty_sky()
src.apply_optical_train(opt, fpa)
hdu = fpa.read_out(OBS_EXPTIME=60 * t, FPA_LINEARITY_CURVE=None)
im = hdu[0].data
stats += [[t, np.sum(im), np.median(im), np.std(im)]]
stats = np.array(stats)
factors = stats / stats[0, :]
bg_stats = [
i == np.round(l**2) == np.round(j) == np.round(k)
for i, j, k, l in factors
]
return_val = np.all(bg_stats)
if not return_val:
print(factors)
return return_val
def test_load_UserCommands_with_invalid_sim_data_dir():
import simmetis as sim
with pytest.raises(FileNotFoundError):
cmd = sim.UserCommands(sim_data_dir="/")
def test_load_UserCommands_with_no_arguments():
import simmetis as sim
with pytest.raises(ValueError):
cmd = sim.UserCommands()
def __init__(self, filename, config, lambda0=0., verbose=False):
'''Read datacube and WCS'''
self.wide_figsize = matplotlib.rcParams['figure.figsize'].copy()
self.wide_figsize[0] = self.wide_figsize[1]*16./9.
self.verbose = verbose
# separate our output from the bullshit printed by import
print('============================================')
self.filename = filename
print("Source file ", filename)
self.src_fits = fits.open(filename)
self.src_cube = self.src_fits[0].data
self.src_header = self.src_fits[0].header
naxis3, naxis2, naxis1 = self.src_cube.shape
print(naxis1, "x", naxis2, "pixels,", naxis3, "spectral channels")
self.plotpix = np.asarray((naxis2//2, naxis1//2))
# Parse the WCS keywords in primary HDU
#del self.src_header['VELREF'] # no AIPS stuff, please
self.wcs = wcs.WCS(self.src_header)
pixscale1, pixscale2 = wcs.utils.proj_plane_pixel_scales(self.wcs)[0:2]
pixscale1 = (pixscale1 * self.wcs.wcs.cunit[0]).to(u.mas)
pixscale2 = (pixscale2 * self.wcs.wcs.cunit[1]).to(u.mas)
#print("image pixscale from WCS:", pixscale1, pixscale2, "/pixel")
self.src_pixscale = np.asarray((pixscale1.value, pixscale2.value))*u.mas
if self.verbose:
print("image pixscale:", self.src_pixscale, "per pixel")
# Check flux units of data cube
try:
bunit = self.src_header['BUNIT']
print("BUNIT:", bunit)
if bunit.lower() == 'jy/pixel':
#plt.semilogy(self.src_cube[:,111,100])
#print(np.max(self.src_cube[:,111,100]))
pixarea = (wcs.utils.proj_plane_pixel_area(self.wcs) *
self.wcs.wcs.cunit[0] *
self.wcs.wcs.cunit[1]).to(u.arcsec*u.arcsec)
if self.verbose:
print("Pixel area is", pixarea)
self.src_cube /= pixarea.value
#plt.semilogy(self.src_cube[:,111,100])
#plt.show()
#print(np.max(self.src_cube[:,111,100]))
except KeyError:
print("Keyword BUNIT not found. Assuming the data is in units of Jy/arcsec2")
# make a Nlambda x 3 array
crd = np.zeros((naxis3, 3))
crd[:, 2] = np.arange(naxis3)
# Convert pixel coordinates to world coordinates
# Second argument is "origin" -- in this case 0-based coordinates
world_coo = self.wcs.wcs_pix2world(crd, 0)[:, 2] * self.wcs.wcs.cunit[2]
if self.verbose:
print("CTYPES:", self.wcs.wcs.ctype)
print("CUNITS:", self.wcs.wcs.cunit)
if self.wcs.wcs.ctype[2] == 'VRAD' or self.wcs.wcs.ctype[2] == 'VOPT':
print("WARNING: Your Fits header specifies the spectral axis as",
self.wcs.wcs.ctype[2])
print(" We treat it as apparent radial velocity (VELO)")
self.wcs.wcs.ctype[2] = 'VELO'
if self.wcs.wcs.ctype[2] == 'VELO':
#print("Velocities:",world_coo)
# this should be in the header, shouldn't it?
if lambda0 > 0.:
restcoo = lambda0 * u.um # unit of lambda0 must be micron
elif self.wcs.wcs.restwav > 0:
restcoo = (self.wcs.wcs.restwav * u.m).to(u.um)
elif self.wcs.wcs.restfrq > 0:
restcoo = (self.wcs.wcs.restfrq * u.Hz).to(u.um, equivalencies=u.spectral())
else:
raise RuntimeError("Cannot determine rest wavelength of velocity scale.")
wavelen = restcoo * (1. + world_coo / const.c)
#print("Wavelengths:",wavelen)
elif self.wcs.wcs.ctype[2] == 'WAVE':
# TODO: test me!
wavelen = world_coo.to(u.um)
restcoo = self.wcs.wcs.restwav * self.wcs.wcs.cunit[2]
elif self.wcs.wcs.ctype[2] == 'FREQ':
wavelen = world_coo.to(u.um, equivalencies=u.spectral())
restcoo = self.wcs.wcs.restfrq * self.wcs.wcs.cunit[2]
else:
print("spectral axis is '", self.wcs.wcs.ctype[2], "'")
raise NotImplementedError('spectral axis must have type VELO, WAVE, or FREQ')
self.wavelen = wavelen.value # should we store it with units?
self.restcoo = restcoo # RESTFRQ or RESTWAV with units
if self.verbose:
print("Wavelengths:", wavelen[0:3], "...", wavelen[-1])
print("restfrq:", self.wcs.wcs.restfrq)
print("restwav:", self.wcs.wcs.restwav)
print("restcoo:", self.restcoo)
#print("Rest wavelen:", restcoo.to(u.um, equivalencies=u.spectral()))
in_velos = wavelen.to(u.m/u.s, equivalencies=u.doppler_optical(restcoo))
step = 1.5 * u.km/u.s
new3 = int((in_velos[-1] - in_velos[0]) / step)+1
meanv = (in_velos[0] + in_velos[-1]) / 2.
self.det_velocities = np.arange((meanv-step*(new3-1)/2.).to(u.m/u.s).value,
(meanv+step*((new3-1)/2.+1)).to(u.m/u.s).value,
step.to(u.m/u.s).value)
if self.verbose:
print("Source velocities (WCS):", in_velos[0], "...", in_velos[-1])
#print(in_velos)
print("new naxis3:", new3)
#print("middle v:", meanv)
print("Detector velocities:", self.det_velocities.shape,
self.det_velocities[0], "...", self.det_velocities[-1])
# initialize the target cube, in case someone does not call transmission_emission()
self.target_cube = self.src_cube
self.target_hdr = self.src_header.copy()
self.target_hdr['BUNIT'] = 'Jy/arcsec2'
self.background = None
self.transmission = 1.
self.emission = None # will be computed later
# initialize SimCADO to set searchpaths
#
print("Reading config", config)
self.cmds = sm.UserCommands(config)
self.det_pixscale = self.cmds["SIM_DETECTOR_PIX_SCALE"] * 1000. # in mas/pixel
if self.verbose:
print("Detector pixel scale ", self.det_pixscale, " mas/pixel")
print("Filter = ", self.cmds["INST_FILTER_TC"]) # should be open filter
if np.max(self.src_pixscale.value) > self.det_pixscale:
print("+------------------------------------------------------------------------------+")
print("| WARNING: MAX. PIXEL SCALE OF INPUT CUBE IS LARGER THAN DETECTOR PIXEL SCALE! |")
print("|", np.max(self.src_pixscale), "/pixel >",
self.det_pixscale, " mas/pixel")
print("+------------------------------------------------------------------------------+")
def mimic_image(hdu,
catalogue,
cmds=None,
hdu_ext=0,
sim_chip_n=0,
return_stamps=False,
cat_ra_name="RA",
cat_dec_name="DE",
cat_filter_name="J",
**kwargs):
"""
Create a SimMETIS image to mimic a real FITS file
Parameters
----------
hdu : str, astropy.HDUList
The original FITS object- either a filename or an astropy.HDUList object
catalogue : str, astropy.Table
A catalogue of stars - either a filename or an astropy.Table object
cmds : str, simmetis.UserCommands
Commands by which to simulate the image - a filename to a .config file or a UserCommands object
hdu_ext : int
The extension in the original FITS file which should be simulated
sim_chip_n : int
Which chip in the FPA_LAYOUT to simulate. Passed to apply_optical_train() and read_out()
return_stamps : bool
If True, returns two PostageStamps object for the original HDU and the generated HDU
cat_ra_name, cat_dec_name, cat_filter_name : str
The names of the column in catalogue which point to the RA, Dec and filter magnitudes for the stars
Optional Parameters
-------------------
As passed to a PostageStamps object
"dx" : 0,
"dy " : 0,
"bg_tile_size" : 24,
"stamp_width" : 24,
"hot_pixel_threshold" : 3000
Returns
-------
hdu_sim, src [, post_real, post_sim]
If return_stamps is True, post_real and post_sim are returned
Examples
--------
::
>>> hdu_real = fits.open("HAWKI.2008-11-05T04_08_55.552.fits")
>>> cat = ascii.read("ngc362_cohen.dat")
>>>
>>> dx, dy = -5, 15
>>>
>>> cmd = sim.UserCommands("hawki_ngc362.config")
>>> cmd["INST_FILTER_TC"] = "J"
>>> cmd["OBS_EXPTIME"] = 10
>>>
>>> out = mimic_image(hdu=hdu_real, catalogue=cat, cmds=cmd, hdu_ext=3,
... sim_chip_n=3, return_stamps=True,
... dx=dx, dy=dy)
>>> hdu_sim, src, post_real, post_sim = out
>>> len(out)
4
"""
if isinstance(hdu, str) and os.path.exists(hdu):
hdu = fits.open(hdu)[hdu_ext]
elif isinstance(hdu, fits.HDUList):
hdu = hdu[hdu_ext]
else:
raise ValueError("hdu must be a filename or an astropy HDU object: " +
type(hdu))
if isinstance(catalogue, str) and os.path.exists(catalogue):
cat = ascii.read(catalogue)
elif isinstance(catalogue, Table):
cat = catalogue
else:
raise ValueError(
"catalogue must be a filename or an astropy.Table object: " +
type(catalogue))
if isinstance(cmds, str) and os.path.exists(cmds):
cmds = sim.UserCommands(cmds)
elif isinstance(cmds, sim.UserCommands):
pass
else:
raise ValueError(
"cmds must be a filename or an simmetis.UserCommands object: " +
type(cmds))
fig = plt.figure(figsize=(0.1, 0.1))
apl_fig = aplpy.FITSFigure(hdu, figure=fig)
# get the RA DEC position of the centre of the HAWKI FoV
xc, yc = hdu.header["CRPIX1"], hdu.header["CRPIX2"]
ra_cen, dec_cen = apl_fig.pixel2world(xc, yc)
# get the x,y positions in arcsec from the HAWKI FoV centre
y = (cat[cat_dec_name] - dec_cen) * 3600
x = -(cat[cat_ra_name] - ra_cen) * 3600 * np.cos(cat[cat_dec_name] / 57.3)
mag = cat[cat_filter_name]
# make a Source object with the x,y positions in arcsec from the HAWKI FoV centre
src = sim.source.stars(mags=mag, filter_name=cat_filter_name, x=x, y=y)
opt = sim.OpticalTrain(cmds)
fpa = sim.Detector(cmds, small_fov=False)
print(sim_chip_n)
src.apply_optical_train(opt, fpa, chips=sim_chip_n)
hdu_sim = fpa.read_out(chips=sim_chip_n)
## Get the Postage Stamps
if return_stamps:
params = {
"dx": 0,
"dy ": 0,
"bg_tile_size": 24,
"stamp_width": 24,
"hot_pixel_threshold": 3000
}
params.update(**kwargs)
w, h = hdu_sim[0].data.shape
mask = (src.x_pix > 0) * (src.x_pix < w) * (src.y_pix > 0) * (src.y_pix
< h)
xw = cat[cat_ra_name][mask]
yw = cat[cat_dec_name][mask]
mag = cat[cat_filter_name][mask]
# get the x,y pixel positions of the stars in the simulated image
xps = src.x_pix[mask]
yps = src.y_pix[mask]
# get the x,y pixel positions of the stars in the real image, include offset if needed
xpr, ypr = apl_fig.world2pixel(xw, yw)
xpr += params["dx"]
ypr += params["dy"]
# get the images from the FITS objects
im_sim = np.copy(hdu_sim[0].data)
im_sim -= np.median(im_sim)
post_sim = PostageStamps(im_sim, x=xps, y=yps, **params)
im_real = np.copy(hdu.data)
im_real -= np.median(im_real)
post_real = PostageStamps(im_real, x=xpr, y=ypr, **params)
return hdu_sim, src, post_real, post_sim
else:
return hdu_sim, src