def fit_trace(self, pos, amp):
info(' Adding fitted aperture near position {:.0f}'.format(pos))
id = len(self.apertures)
g0 = models.Gaussian1D(mean=pos, amplitude=amp,
bounds={'amplitude': [0, float('+Inf')]})
fitter = fitting.LevMarLSQFitter()
g = fitter(g0, self.xdata, self.ydata)
# Set maximum width of aperture to the YOFFSET parameter
try:
maxap = np.floor(float(self.header['YOFFSET'])/0.1799)
except:
maxap = self.data.shape[0]
# Set minimum width of aperture to an estimate of 3x the seeing
seeing = 0.5 # arcsec
minap = np.ceil(3*seeing/0.1799)
width = np.ceil(5.*g.param_sets[2][0])
width = max(min(maxap, width), minap)
data = {'id': id,
'position': g.param_sets[1][0],
'width': width,
'amplitude': g.param_sets[0][0],
'sigma': g.param_sets[2][0],
}
self.apertures.add_row(data)
self.plot_apertures()
def keypress(self, event):
'''Based on which key is presses on a key press event, call the
appropriate method.
'''
if event.key == 'a':
if py3:
response = input("Please enter new half width in pixels: ")
else:
response = raw_input("Please enter new half width in pixels: ")
width = int(response)
self.add_aperture(event.xdata, width)
print('Adding aperture at position {}, width {}'.format(event.xdata,
width))
elif event.key == 'w':
id = self.determine_id(event)
self.set_width(id=id)
elif event.key == 'p':
id = self.determine_id(event)
self.set_position(id=id, pos=event.xdata)
elif event.key == 'g':
self.fit_trace(event.xdata, event.ydata/abs(event.ydata))
elif event.key == 'd':
id = self.determine_id(event)
self.delete_aperture(id)
elif event.key == 'n':
self.quit(event)
elif event.key == 'q':
self.quit(event)
elif event.key == 's':
print(self.apertures)
elif event.key == 'r':
info(' Plotting apertures')
self.plot_apertures()
def handle_file_list(output_file, files):
'''Write a list of paths to MOSFIRE file to output_file.'''
if os.path.isfile(output_file):
print("%s: already exists, skipping" % output_file )
# pass
if len(files) > 0:
with open(output_file, "w") as f:
f = open(output_file, "w")
f.write(descriptive_blurb())
picker = lambda x: x
if len(files[0]) == 2: picker = lambda x: x[0]
# Identify unique path to files:
paths = [os.path.dirname(picker(file)) for file in files]
paths = list(set(paths))
if len(paths) == 1:
path_to_all = paths[0]
converter = os.path.basename
f.write("%s # Abs. path to files [optional]\n" % path_to_all)
else:
converter = lambda x: x
info('Writing {} files to {}'.format(len(files), output_file))
for path in files:
if len(path) == 2:
to_write = "%s # %s s\n" % (converter(path[0]), path[1])
else:
to_write = "%s\n" % converter(path)
f.write("%s" % to_write)
def handle_file_list(output_file, files):
'''Write a list of paths to MOSFIRE file to output_file.'''
if os.path.isfile(output_file):
print("%s: already exists, skipping" % output_file)
# pass
if len(files) > 0:
with open(output_file, "w") as f:
f = open(output_file, "w")
f.write(descriptive_blurb())
picker = lambda x: x
if len(files[0]) == 2: picker = lambda x: x[0]
# Identify unique path to files:
paths = [os.path.dirname(picker(file)) for file in files]
paths = list(set(paths))
if len(paths) == 1:
path_to_all = paths[0]
converter = os.path.basename
f.write("%s # Abs. path to files [optional]\n" % path_to_all)
else:
converter = lambda x: x
info('Writing {} files to {}'.format(len(files), output_file))
for path in files:
if len(path) == 2:
to_write = "%s # %s s\n" % (converter(path[0]), path[1])
else:
to_write = "%s\n" % converter(path)
f.write("%s" % to_write)
def guess(self):
## Try to guess at aperture positions if no information given
## Start with maximum pixel value
valid_profile = list(self.ydata[~self.ydata.mask])
maxval = max(valid_profile)
maxind = valid_profile.index(maxval)
info(' Guessing at aperture near position {}'.format(maxind))
return (maxind, maxval)
def set_header(self, header, ssl=None, msl=None, asl=None, targs=None):
'''Passed "header" a FITS header dictionary and converts to a Barset'''
self.pos = np.array(IO.parse_header_for_bars(header))
self.set_pos_pix()
self.ssl = ssl
self.msl = msl
self.asl = asl
self.targs = targs
def is_alignment_slit(slit):
return (np.float(slit["Target_Priority"]) < 0)
# If len(ssl) == 0 then the header is for a long slit
if (header['MASKNAME'] == 'long2pos'):
info("long2pos mode in CSU slit determination")
self.long2pos_slit = True
if (len(ssl) == 0):
self.long_slit = True
start = np.int(msl[0]["Slit_Number"])
stop = np.int(msl[-1]["Slit_Number"])
for mech_slit in msl:
mech_slit["Target_in_Slit"] = "long"
self.ssl = np.array([("1", "??", "??", "??", "??", "??", "??", msl[0]['Slit_width'],
(stop-start+1)*7.6, "0", "long", "0")],
dtype= [ ('Slit_Number', '|S2'),
('Slit_RA_Hours', '|S2'), ('Slit_RA_Minutes', '|S2'), ('Slit_RA_Seconds', '|S5'),
('Slit_Dec_Degrees', '|S3'), ('Slit_Dec_Minutes', '|S2'), ('Slit_Dec_Seconds', '|S5'),
('Slit_width', '|S5'), ('Slit_length', '|S5'), ('Target_to_center_of_slit_distance', '|S5'),
('Target_Name', '|S80'), ('Target_Priority', '|S1')])
self.scislit_to_slit = [ np.arange(start,stop) ]
ssl = None
# Create a map between scislit number and mechanical slit
# recall that slits count from 1
if ssl is not None:
prev = self.msl[0]["Target_in_Slit"]
v = []
for science_slit in ssl:
targ = science_slit["Target_Name"]
v.append([int(x) for x in self.msl.field("Slit_Number")[np.where(self.msl.field("Target_in_Slit").rstrip() == targ)[0]]])
self.scislit_to_slit = v
if (len(self.scislit_to_slit) != len(ssl)) and not (self.long_slit
and len(self.scislit_to_slit) == 1):
error("SSL should match targets in slit")
raise Exception("SSL should match targets in slit")
def writefits(img, maskname, fname, options, header=None, bs=None,
overwrite=False, lossy_compress=False):
'''Convenience wrapper to write MOSFIRE drp-friendly FITS files
Args:
img: Data array to write to disk
maskname: Name of the science mask
fname: Full or relative path to output file
options: {} Unused
header: Optional, the header to write
bs: Optional unused
overwrite: Force overwrite of file, default False/No.
lossy_compress: Zero out the lowest order bits of the floats in
order to make FITS files amenable to compression. The loss is
at least 10 x less than 5e- which is the lowest reasonable read-
noise value.
Results:
Writes a file to fname with data img and header header.
'''
if lossy_compress:
hdu = pf.PrimaryHDU(floatcompress(img))
else:
hdu = pf.PrimaryHDU(img)
fn = fname
if header is None: header = {"DRPVER": (MOSFIRE.__version__, "DRP Version Date")}
else: header["DRPVER"] = (MOSFIRE.__version__, 'DRP Version Date')
warnings.filterwarnings('ignore')
if header is not None:
for k,value, comment in header.cards:
if k in hdu.header: continue
if k == 'COMMENT': continue
if k == '': continue
k = k.rstrip()
hdu.header[k] = (value,comment)
warnings.filterwarnings('always')
if overwrite:
try:
os.remove(fn)
debug("Removed old file '{0}'".format(fn))
except: pass
info("Wrote to '%s'" % (fn))
warnings.filterwarnings('ignore','Card is too long, comment will be truncated.')
hdu.writeto(fn)
warnings.filterwarnings('always')
if lossy_compress: os.system("gzip --force {0}".format(fn))
def extract_spectra(maskname, band, interactive=True, width=10,
target='default'):
if target == 'default':
## Get objectnames from slit edges
edges = np.load('slit-edges_{}.npy'.format(band))
objectnames = [edge['Target_Name'] for edge in edges[:-1]]
eps_files = ['{}_{}_{}_eps.fits'.format(maskname, band, objectname)
for objectname in objectnames]
sig_files = ['{}_{}_{}_sig.fits'.format(maskname, band, objectname)
for objectname in objectnames]
spectrum_plot_files = ['{}_{}_{}.png'.format(maskname, band, objectname)
for objectname in objectnames]
fits_files = ['{}_{}_{}_1D.fits'.format(maskname, band, objectname)
for objectname in objectnames]
else:
objectnames = [target]
eps_files = ['{}_{}_eps.fits'.format(target, band)]
sig_files = ['{}_{}_sig.fits'.format(target, band)]
spectrum_plot_files = ['{}_{}.png'.format(target, band)]
fits_files = ['{}_{}_1D.fits'.format(target, band)]
aperture_tables = {}
if interactive:
print_instructions()
# First, iterate through all slits and define the apertures to extract
for i,eps_file in enumerate(eps_files):
objectname = objectnames[i]
eps = fits.open(eps_file, 'readonly')[0]
aperture_tables[objectname] = find_apertures(eps, width=width, title=objectname,
interactive=interactive,
maskname=maskname,
)
# Second, iterate through all slits again and perform spectral extraction
# using the apertures defined above
for i,eps_file in enumerate(eps_files):
objectname = objectnames[i]
sig_file = sig_files[i]
spectrum_plot_file = spectrum_plot_files[i]
fits_file = fits_files[i]
if len(aperture_tables[objectname]) == 0:
info('No apertures defined for {}. Skipping extraction.'.format(
objectname))
else:
info('Extracting spectra for {}'.format(objectname))
eps = fits.open(eps_file, 'readonly')[0]
sig = fits.open(sig_file, 'readonly')[0]
try:
optimal_extraction(eps, sig, aperture_tables[objectname],
fitsfileout=fits_file,
plotfileout=spectrum_plot_file,
)
except Exception as e:
warning('Failed to extract spectra for {}'.format(objectname))
warning(e)
def fix_long2pos_headers(filelist):
'''Fixes old long2pos observations which have a wrong set of keywords'''
files = list_file_to_strings(filelist)
# Print the filenames to Standard-out
info("Fixing long2pos headers for files in "+str(filelist))
# Iterate through files
for fname in files:
if os.path.isabs(fname): path = fname
else: path = os.path.join(fname_to_path(fname, options), fname)
hdulist = pf.open(path, mode='update')
header = hdulist[0].header
# determine if this file really needs to be updated (for example,
# prevents a second update of an already updated file
if 'long2pos' in header['MASKNAME'] and header['FRAMEID']=='object'\
and (header['PATTERN']=='long2pos' or header['PATTERN']=='Stare'):
info( "File "+str(fname)+" will be updated")
# make a copy of the original file
newname = path+".original"
info("copying ... "+str(path))
info("into ...... "+str(newname))
shutil.copyfile(path,newname)
if not os.path.exists(newname):
errstr = "Error in generating original file: '%s' does not exist"\
"(could not be created)." % newname
error(errstr)
raise Exception(errstr)
#updating header
# assign FRAMEID to narrow slits
if header['YOFFSET']==21 or header['YOFFSET']==-7:
header['FRAMEID']="B"
if header['YOFFSET']==-21 or header['YOFFSET']==7:
header['FRAMEID']="A"
# assign FRAMEID to wide slits
if header['YOFFSET']==14 or header['YOFFSET']==-14:
header['FRAMEID']="A"
#reverse sign of offsets for narrow slits
if header['YOFFSET']==-21:
header['YOFFSET']=7
if header['YOFFSET']==21:
header['YOFFSET']=-7
#transform Xoffset from pixels to arcseconds
header['XOFFSET'] = header['XOFFSET']*0.18
else:
info("File "+str(fname)+" does not need to be updated")
hdulist.flush()
hdulist.close()
def add_aperture(self, pos, width):
info(' Adding aperture at position {} of width {}'.format(pos, width))
id = len(self.apertures)
data = {'id': id,
'position': pos,
'width': width,
'amplitude': None,
'sigma': None,
}
self.apertures.add_row(data)
self.plot_apertures()
def fix_long2pos_headers(filelist):
'''Fixes old long2pos observations which have a wrong set of keywords'''
files = list_file_to_strings(filelist)
# Print the filenames to Standard-out
info("Fixing long2pos headers for files in " + str(filelist))
# Iterate through files
for fname in files:
if os.path.isabs(fname): path = fname
else: path = os.path.join(fname_to_path(fname, options), fname)
hdulist = pf.open(path, mode='update')
header = hdulist[0].header
# determine if this file really needs to be updated (for example,
# prevents a second update of an already updated file
if 'long2pos' in header['MASKNAME'] and header['FRAMEID']=='object'\
and (header['PATTERN']=='long2pos' or header['PATTERN']=='Stare'):
info("File " + str(fname) + " will be updated")
# make a copy of the original file
newname = path + ".original"
info("copying ... " + str(path))
info("into ...... " + str(newname))
shutil.copyfile(path, newname)
if not os.path.exists(newname):
errstr = "Error in generating original file: '%s' does not exist"\
"(could not be created)." % newname
error(errstr)
raise Exception(errstr)
#updating header
# assign FRAMEID to narrow slits
if header['YOFFSET'] == 21 or header['YOFFSET'] == -7:
header['FRAMEID'] = "B"
if header['YOFFSET'] == -21 or header['YOFFSET'] == 7:
header['FRAMEID'] = "A"
# assign FRAMEID to wide slits
if header['YOFFSET'] == 14 or header['YOFFSET'] == -14:
header['FRAMEID'] = "A"
#reverse sign of offsets for narrow slits
if header['YOFFSET'] == -21:
header['YOFFSET'] = 7
if header['YOFFSET'] == 21:
header['YOFFSET'] = -7
#transform Xoffset from pixels to arcseconds
header['XOFFSET'] = header['XOFFSET'] * 0.18
else:
info("File " + str(fname) + " does not need to be updated")
hdulist.flush()
hdulist.close()
def test_lhs_v_rhs(self):
sa = self.assertTrue
p0 = [0.5, 5, 1.1, .7, 0]
pn0 = [0.5, 5, -1.1, .7, 0]
xs = np.arange(25)
ys = fit_single(p0, xs)
lsf = do_fit(ys, residual_single)
info(str(lsf[0]))
ys = fit_single(pn0, xs)
lsf = do_fit(ys, residual_single)
info(str(lsf[0]))
def set_width(self, id=None, width=None):
assert id is not None
info(' Changing width for aperture {} at position {:.0f}'.format(id,
self.apertures[id]['position']))
if width:
self.apertures[id]['width'] = width
else:
if py3:
response = input("Please enter new half width in pixels: ")
else:
response = raw_input("Please enter new half width in pixels: ")
width = int(response)
self.apertures[id]['width'] = width
self.plot_apertures()
def set_position(self, id=None, pos=None):
assert id is not None
info(' Moving aperture {} from {:.0f} to {:.0f}'.format(id,
float(self.apertures[id]['position']), float(pos)))
if pos:
self.apertures[id]['position'] = pos
else:
if py3:
response = input("Please enter new position in pixels: ")
else:
response = raw_input("Please enter new position in pixels: ")
pos = int(response)
self.apertures[id]['position'] = pos
self.plot_apertures()
def iterate_spatial_profile(P, DmS, V, f,\
smoothing=5, order=3, minpixels=50,\
sigma=4, nclip=2, verbose=True):
poly0 = models.Polynomial1D(degree=order)
fit_poly = fitting.LinearLSQFitter()
Pnew = np.zeros(P.shape)
for i,row in enumerate(P):
weights = f**2/V[i]
weights.mask = weights.mask | np.isnan(weights)
srow = np.ma.MaskedArray(data=signal.medfilt(row, smoothing),\
mask=(np.isnan(signal.medfilt(row, smoothing)) | weights.mask))
xcoord = np.ma.MaskedArray(data=np.arange(0,len(row),1),\
mask=srow.mask)
for iter in range(nclip+1):
nrej_before = np.sum(srow.mask)
fitted_poly = fit_poly(poly0,\
xcoord[~xcoord.mask], srow[~srow.mask],\
weights=weights[~weights.mask])
fit = np.array([fitted_poly(x) for x in xcoord])
resid = (DmS[i]-f*srow)**2 / V[i]
newmask = (resid > sigma)
weights.mask = weights.mask | newmask
srow.mask = srow.mask | newmask
xcoord.mask = xcoord.mask | newmask
nrej_after = np.sum(srow.mask)
if nrej_after > nrej_before:
if verbose:\
info('Row {:3d}: Rejected {:d} pixels on clipping '+\
'iteration {:d}'.format(\
i, nrej_after-nrej_before, iter))
## Reject row if too few pixels are availabel for the fit
if (srow.shape[0] - nrej_after) < minpixels:
if verbose:
warning('Row {:3d}: WARNING! Only {:d} pixels remain after '+\
'clipping and masking'.format(\
i, srow.shape[0] - nrej_after))
fit = np.zeros(fit.shape)
## Set negative values to zero
if np.sum((fit<0)) > 0 and verbose:
info('Row {:3d}: Reset {:d} negative pixels in fit to 0'.format(\
i, np.sum((fit<0))))
fit[(fit < 0)] = 0
Pnew[i] = fit
return Pnew
def fit_edge_poly(xposs, xposs_missing, yposs, order):
'''
fit_edge_poly fits a polynomial to the measured slit edges.
This polynomial is used to extract spectra.
fit_edge_poly computes a parabola, and fills in missing data with a
parabola
input-
xposs, yposs [N]: The x and y positions of the slit edge [pix]
order: the polynomial order
'''
# First fit low order polynomial to fill in missing data
fun = np.poly1d(Fit.polyfit_clip(xposs, yposs, 2))
xposs = np.append(xposs, xposs_missing)
yposs = np.append(yposs, fun(xposs_missing))
# Remove any fits that deviate wildly from the 2nd order polynomial
ok = np.abs(yposs - fun(xposs)) < 1
if not ok.any():
error("Flat is not well illuminated? Cannot find edges")
raise Exception("Flat is not well illuminated? Cannot find edges")
# Now refit to user requested order
fun = np.poly1d(Fit.polyfit_clip(xposs[ok], yposs[ok], order))
yposs_ok = yposs[ok]
res = fun(xposs[ok]) - yposs[ok]
sd = np.std(res)
ok = np.abs(res) < 2*sd
# Check to see if the slit edge funciton is sane,
# if it's not, then we fix it.
pix = np.arange(2048)
V = fun(pix)
if np.abs(V.max() - V.min()) > 10:
info ("Forcing a horizontal slit edge")
print("Forcing a horizontal slit edge")
tmp = yposs_ok[ok]
fun = np.poly1d(np.median(tmp))
#fun = np.poly1d(np.median(yposs[ok]))
return (fun, res, sd, ok)
def __init__(self, hdu, title=None, interactive=False):
self.header = hdu.header
self.data = np.ma.MaskedArray(data=hdu.data, mask=np.isnan(hdu.data))
self.ydata = np.mean(self.data, axis=1)
self.xdata = list(range(0,len(self.ydata), 1))
self.interactive = interactive
if self.interactive:
self.fig = pl.figure()
self.ax = self.fig.gca()
self.title = title
if title:
info('Editing apertures for {}'.format(title))
else:
info('Editing apertures')
self.apertures = table.Table(names=('id', 'position', 'width',\
'amplitude', 'sigma'),\
dtype=('i4', 'f4', 'f4', 'f4', 'f4'))
def fit_edge_poly(xposs, xposs_missing, yposs, order):
'''
fit_edge_poly fits a polynomial to the measured slit edges.
This polynomial is used to extract spectra.
fit_edge_poly computes a parabola, and fills in missing data with a
parabola
input-
xposs, yposs [N]: The x and y positions of the slit edge [pix]
order: the polynomial order
'''
# First fit low order polynomial to fill in missing data
fun = np.poly1d(Fit.polyfit_clip(xposs, yposs, 2))
xposs = np.append(xposs, xposs_missing)
yposs = np.append(yposs, fun(xposs_missing))
# Remove any fits that deviate wildly from the 2nd order polynomial
ok = np.abs(yposs - fun(xposs)) < 1
if not ok.any():
error("Flat is not well illuminated? Cannot find edges")
raise Exception("Flat is not well illuminated? Cannot find edges")
# Now refit to user requested order
fun = np.poly1d(Fit.polyfit_clip(xposs[ok], yposs[ok], order))
yposs_ok = yposs[ok]
res = fun(xposs[ok]) - yposs[ok]
sd = np.std(res)
ok = np.abs(res) < 2 * sd
# Check to see if the slit edge funciton is sane,
# if it's not, then we fix it.
pix = np.arange(2048)
V = fun(pix)
if np.abs(V.max() - V.min()) > 10:
info("Forcing a horizontal slit edge")
print("Forcing a horizontal slit edge")
tmp = yposs_ok[ok]
fun = np.poly1d(np.median(tmp))
#fun = np.poly1d(np.median(yposs[ok]))
return (fun, res, sd, ok)
def find_apertures(hdu, guesses=[], width=10, title=None, interactive=True,
maskname=''):
'''Finds targets in spectra by simply collapsing the 2D spectra in the
wavelength direction and fitting Gaussian profiles to the positional profile
'''
pl.ioff()
ap = ApertureEditor(hdu, title=title, interactive=interactive)
if interactive:
ap.connect()
if re.search('LONGSLIT', maskname):
guesses = [int(np.argmax(ap.ydata))]
if (guesses == []) or (guesses is None):
# Guess at object position where specified in header by CRVAL2
pos = int(-hdu.header['CRVAL2'])
amp = ap.ydata[int(pos)]
# Estimate signal to noise of profile at that spot
# First, clip top and bottom 10 percent of pixels to roughly remove
# contribution by a single bright source.
pct = 10
pctile = (np.percentile(ap.ydata, pct), np.percentile(ap.ydata, 100-pct))
w = np.where((ap.ydata > pctile[0]) & (ap.ydata < pctile[1]))
# Use the unclipped pixels to estimate signal to noise.
std = np.std(ap.ydata[w])
snr = amp/std
# If SNR is strong, fit a gaussian, if not, just blindly add an aperture
if snr > 5:
info(' Using trace to determine extraction aperture')
ap.fit_trace(pos, amp)
else:
info(' Could not find strong trace, adding aperture blindly')
ap.add_aperture(pos, width)
else:
for guess in guesses:
ap.fit_trace(guess, ap.ydata[guess])
return ap.apertures
def imarith(operand1, op, operand2, result):
info( "%s %s %s -> %s" % (operand1, op, operand2, result))
assert type(operand1) == str
assert type(operand2) == str
assert os.path.exists(operand1)
assert os.path.exists(operand2)
assert op in ['+', '-']
## Strip off the [0] part of the operand as we are assuming that we are
## operating on the 0th FITS HDU.
if re.match('(\w+\.fits)\[0\]', operand1):
operand1 = operand1[:-3]
if re.match('(\w+\.fits)\[0\]', operand2):
operand2 = operand2[:-3]
import operator
operation = { "+": operator.add, "-": operator.sub,\
"*": operator.mul, "/": operator.truediv}
hdulist1 = pf.open(operand1, 'readonly')
hdulist2 = pf.open(operand2, 'readonly')
data = operation[op](hdulist1[0].data, hdulist2[0].data)
header = hdulist1[0].header
header['history'] = 'imarith {} {} {}'.format(operand1, op, operand2)
header['history'] = 'Header values copied from {}'.format(operand1)
if 'EXPTIME' in hdulist1[0].header and\
'EXPTIME' in hdulist2[0].header and\
operation in ['+', '-']:
exptime = operation[op](float(hdulist1[0].header['EXPTIME']),\
float(hdulist2[0].header['EXPTIME']))
header['history'] = 'Other than exposure time which was edited'
header['EXPTIME'] = exptime
hdu = pf.PrimaryHDU(data=data, header=header)
hdu.writeto(result)
hdulist1.close()
hdulist2.close()
def imarith(operand1, op, operand2, result):
info("%s %s %s -> %s" % (operand1, op, operand2, result))
assert type(operand1) == str
assert type(operand2) == str
assert os.path.exists(operand1)
assert os.path.exists(operand2)
assert op in ['+', '-']
## Strip off the [0] part of the operand as we are assuming that we are
## operating on the 0th FITS HDU.
if re.match('(\w+\.fits)\[0\]', operand1):
operand1 = operand1[:-3]
if re.match('(\w+\.fits)\[0\]', operand2):
operand2 = operand2[:-3]
import operator
operation = { "+": operator.add, "-": operator.sub,\
"*": operator.mul, "/": operator.truediv}
hdulist1 = pf.open(operand1, 'readonly')
hdulist2 = pf.open(operand2, 'readonly')
data = operation[op](hdulist1[0].data, hdulist2[0].data)
header = hdulist1[0].header
header['history'] = 'imarith {} {} {}'.format(operand1, op, operand2)
header['history'] = 'Header values copied from {}'.format(operand1)
if 'EXPTIME' in hdulist1[0].header and\
'EXPTIME' in hdulist2[0].header and\
operation in ['+', '-']:
exptime = operation[op](float(hdulist1[0].header['EXPTIME']),\
float(hdulist2[0].header['EXPTIME']))
header['history'] = 'Other than exposure time which was edited'
header['EXPTIME'] = exptime
hdu = pf.PrimaryHDU(data=data, header=header)
hdu.writeto(result)
hdulist1.close()
hdulist2.close()
def find_and_fit_edges(data, header, bs, options,edgeThreshold=450):
'''
Given a flat field image, find_and_fit_edges determines the position
of all slits.
The function works by starting with a guess at the location for a slit
edge in the spatial direction(options["first-slit-edge"]).
Starting from the guess, find_edge_pair works out in either direction,
measuring the position of the (e.g.) bottom of slit 1 and top of slit 2:
------ pixel y value = 2048
Slit 1 data
------ (bottom)
deadband
------ (top)
Slit N pixel data ....
------- (bottom) pixel = 0
--------------------------------> Spectral direction
1. At the top of the flat, the slit edge is defined to be a pixel value
2. The code guesses the position of the bottom of the slit, and runs
find_edge_pair to measure slit edge locations.
3. A low-order polynomial is fit to the edge locations with
fit_edge_poly
4. The top and bottom of the current slit, is stored into the
result list.
5. The top of the next slit is stored temporarily for the next
iteration of the for loop.
6. At the bottom of the flat, the slit edge is defined to be pixel 4.
options:
options["edge-order"] -- The order of the polynomial [pixels] edge.
options["edge-fit-width"] -- The length [pixels] of the edge to
fit over
'''
# TODO: move hardcoded values into Options.py
# y is the location to start
y = 2034
DY = 44.25
toc = 0
ssl = bs.ssl
slits = []
top = [0., np.float(Options.npix)]
start_slit_num = int(bs.msl[0]['Slit_Number'])-1
if start_slit_num > 0:
y -= DY * start_slit_num
# Count and check that the # of objects in the SSL matches that of the MSL
# This is purely a safety check
numslits = np.zeros(len(ssl))
for i in range(len(ssl)):
slit = ssl[i]
M = np.where(slit["Target_Name"] == bs.msl["Target_in_Slit"])
numslits[i] = len(M[0])
numslits = np.array(numslits)
if (np.sum(numslits) != CSU.numslits) and (not bs.long_slit) and (not bs.long2pos_slit):
error ("The number of allocated CSU slits (%i) does not match "
" the number of possible slits (%i)." % (np.sum(numslits),
CSU.numslits))
raise Exception("The number of allocated CSU slits (%i) does not match "
" the number of possible slits (%i)." % (np.sum(numslits),
CSU.numslits))
# if the mask is a long slit, the default y value will be wrong. Set instead to be the middle
if bs.long_slit:
y = 1104
# now begin steps outline above
results = []
result = {}
result["Target_Name"] = ssl[0]["Target_Name"]
# 1
result["top"] = np.poly1d([y])
''' Nomenclature here is confusing:
----- Edge -- Top of current slit, bottom of prev slit
. o ' Data
===== Data
.;.;' Data
----- Edge -- Bottom of current slit, top of next slit
'''
topfun = np.poly1d([y])
xposs_top_this = np.arange(10,2000,100)
yposs_top_this = topfun(xposs_top_this)
initial_edges = np.array([2034], dtype=np.int)
edge = 2034
# build an array of values containing the lower edge of the slits
for target in range(len(ssl)):
# target is the slit number
edge -= DY * numslits[target]
initial_edges=np.append(initial_edges,int(edge))
# collapse the 2d flat along the walenegth axis to build a spatial profile of the slits
vertical_profile = np.mean(data, axis=1)
# build an array containing the spatial positions of the slit centers, basically the mid point between the expected
# top and bottom values of the slit pixels
spatial_centers = np.array([], dtype=np.int)
for k in np.arange(0,len(initial_edges)-1):
spatial_centers = np.append(spatial_centers,(initial_edges[k]+initial_edges[k+1])//2)
#slit_values=np.array([])
#for k in np.arange(0, len(spatial_centers)):
# slit_values = np.append(slit_values,np.mean(vertical_profile[spatial_centers[k]-3:spatial_centers[k]+3]))
for target in range(len(ssl)):
y -= DY * numslits[target]
y = max(y, 1)
# select a 6 pixel wide section of the vertical profile around the slit center
threshold_area = vertical_profile[spatial_centers[target]-3:spatial_centers[target]+3]
# uses 80% of the ADU counts in the threshold area to estimate the threshold to use in defining the slits
edgeThreshold = np.mean(threshold_area)*0.8
#if edgeThreshold > 450:
# edgeThreshold = 450
info("[%2.2i] Finding Slit Edges for %s ending at %4.0i. Slit "
"composed of %i CSU slits" % ( target,
ssl[target]["Target_Name"], y, numslits[target]))
info("[%2.2i] Threshold used is %.1f" % (target,edgeThreshold))
''' First deal with the current slit '''
hpps = Wavelength.estimate_half_power_points(
bs.scislit_to_csuslit(target+1)[0], header, bs)
if y == 1:
xposs_bot = [1024]
xposs_bot_missing = []
yposs_bot = [4.25]
botfun = np.poly1d(yposs_bot)
ok = np.where((xposs_bot > hpps[0]) & (xposs_bot < hpps[1]))
else:
(xposs_top_next, xposs_top_next_missing, yposs_top_next, xposs_bot,
xposs_bot_missing, yposs_bot, scatter_bot_this) = find_edge_pair(
data, y, options["edge-fit-width"],edgeThreshold=edgeThreshold)
ok = np.where((xposs_bot > hpps[0]) & (xposs_bot < hpps[1]))
ok2 = np.where((xposs_bot_missing > hpps[0]) & (xposs_bot_missing <
hpps[1]))
xposs_bot = xposs_bot[ok]
xposs_bot_missing = xposs_bot_missing[ok2]
yposs_bot = yposs_bot[ok]
if len(xposs_bot) == 0:
botfun = np.poly1d(y-DY)
else:
(botfun, bot_res, botsd, botok) = fit_edge_poly(xposs_bot,
xposs_bot_missing, yposs_bot, options["edge-order"])
bot = botfun.c.copy()
top = topfun.c.copy()
#4
result = {}
result["Target_Name"] = ssl[target]["Target_Name"]
result["xposs_top"] = xposs_top_this
result["yposs_top"] = yposs_top_this
result["xposs_bot"] = xposs_bot
result["yposs_bot"] = yposs_bot
result["top"] = np.poly1d(top)
result["bottom"] = np.poly1d(bot)
result["hpps"] = hpps
result["ok"] = ok
results.append(result)
#5
if y == 1:
break
next = target + 2
if next > len(ssl): next = len(ssl)
hpps_next = Wavelength.estimate_half_power_points(
bs.scislit_to_csuslit(next)[0],
header, bs)
ok = np.where((xposs_top_next > hpps_next[0]) & (xposs_top_next <
hpps_next[1]))
ok2 = np.where((xposs_top_next_missing > hpps_next[0]) &
(xposs_top_next_missing < hpps_next[1]))
xposs_top_next = xposs_top_next[ok]
xposs_top_next_missing = xposs_top_next_missing[ok2]
yposs_top_next = yposs_top_next[ok]
if len(xposs_top_next) == 0:
topfun = np.poly1d(y)
else:
(topfun, topres, topsd, ok) = fit_edge_poly(xposs_top_next,
xposs_top_next_missing, yposs_top_next, options["edge-order"])
xposs_top_this = xposs_top_next
xposs_top_this_missing = xposs_top_next_missing
yposs_top_this = yposs_top_next
results.append({"version": options["version"]})
return results
def find_longslit_edges(data, header, bs, options, edgeThreshold=450,longslit=None):
y = 2034
DY = 44.25
toc = 0
ssl = bs.ssl
slits = []
top = [0., np.float(Options.npix)]
start_slit_num = int(bs.msl[0]['Slit_Number'])-1
if start_slit_num > 0:
y -= DY * start_slit_num
# if the mask is a long slit, the default y value will be wrong. Set instead to be the middle
if bs.long_slit:
try:
y=longslit["yrange"][1]
except:
error("Longslit reduction mode is specified, but the row position has not been specified. Defaulting to "+str(y))
print("Longslit reduction mode is specified, but the row position has not been specified. Defaulting to "+str(y))
# Count and check that the # of objects in the SSL matches that of the MSL
# This is purely a safety check
numslits = np.zeros(len(ssl))
for i in range(len(ssl)):
slit = ssl[i]
M = np.where(slit["Target_Name"] == bs.msl["Target_in_Slit"])
numslits[i] = len(M[0])
numslits = np.array(numslits)
info("Number of slits allocated for this longslit: "+str(np.sum(numslits)))
# now begin steps outline above
results = []
result = {}
result["Target_Name"] = ssl[0]["Target_Name"]
# 1 Defines a polynomial of degree 0, which is a constant, with the value of the top of the slit
result["top"] = np.poly1d([longslit["yrange"][1]])
topfun = np.poly1d([longslit["yrange"][1]]) # this is a constant funtion with c=top of the slit
botfun = np.poly1d([longslit["yrange"][0]]) # this is a constant funtion with c=bottom of the slit
# xposs_top_this = [10 110 210 .... 1810 1910]
xposs_top = np.arange(10,2000,100)
xposs_bot = np.arange(10,2000,100)
# yposs_top_this = [1104 1104 ... 1104 1104], it's the constant polynomium calculated at the X positions
yposs_top = topfun(xposs_top)
yposs_bot = botfun(xposs_bot)
''' Deal with the current slit '''
target=0
hpps = Wavelength.estimate_half_power_points(
bs.scislit_to_csuslit(target+1)[0], header, bs)
ok = np.where((xposs_top > hpps[0]) & (xposs_top < hpps[1]))
xposs_bot = xposs_bot[ok]
yposs_bot = yposs_bot[ok]
xposs_top = xposs_top[ok]
yposs_top = yposs_top[ok]
if len(xposs_bot) == 0:
error ("The slit edges specifications appear to be incorrect.")
raise Exception("The slit edges specifications appear to be incorrect.")
# bot is the polynomium that defines the shape of the bottom of the slit. In this case, we set it to a constant.
bot = botfun.c.copy()
top = topfun.c.copy()
#4
result = {}
result["Target_Name"] = ssl[target]["Target_Name"]
result["xposs_top"] = xposs_top
result["yposs_top"] = yposs_top
result["xposs_bot"] = xposs_bot
result["yposs_bot"] = yposs_bot
result["top"] = np.poly1d(top)
result["bottom"] = np.poly1d(bot)
result["hpps"] = hpps
result["ok"] = ok
results.append(result)
results.append({"version": options["version"]})
return results
def handle_flats(flatlist, maskname, band, options, extension=None,edgeThreshold=450,lampOffList=None,longslit=None):
'''
handle_flats is the primary entry point to the Flats module.
handle_flats takes a list of individual exposure FITS files and creates:
1. A CRR, dark subtracted, pixel-response flat file.
2. A set of polynomials that mark the edges of a slit
Inputs:
flatlist:
maskname: The name of a mask
band: A string indicating the bandceil
Outputs:
file {maskname}/flat_2d_{band}.fits -- pixel response flat
file {maskname}/edges.np
'''
tick = time.time()
# Check
bpos = np.ones(92) * -1
#Retrieve the list of files to use for flat creation.
flatlist = IO.list_file_to_strings(flatlist)
if len(flatlist) == 0:
print('WARNING: No flat files found.')
raise IOError('No flat files found')
# Print the filenames to Standard-out
for flat in flatlist:
debug(str(flat))
#Determine if flat files headers are in agreement
for fname in flatlist:
hdr, dat, bs = IO.readmosfits(fname, options, extension=extension)
try: bs0
except: bs0 = bs
if np.any(bs0.pos != bs.pos):
print("bs0: "+str(bs0.pos)+" bs: "+str(bs.pos))
error("Barset do not seem to match")
raise Exception("Barsets do not seem to match")
if hdr["filter"] != band:
error ("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
raise Exception("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
for i in range(len(bpos)):
b = hdr["B{0:02d}POS".format(i+1)]
if bpos[i] == -1:
bpos[i] = b
else:
if bpos[i] != b:
error("Bar positions are not all the same in "
"this set of flat files")
raise Exception("Bar positions are not all the same in "
"this set of flat files")
bs = bs0
# Imcombine the lamps ON flats
info("Attempting to combine files in {}".format(flatlist))
out = os.path.join("combflat_2d_{:s}.fits".format(band))
IO.imcombine(flatlist, out, options, reject="minmax", nlow=1, nhigh=1)
# Imcombine the lamps OFF flats and subtract the off from the On sets
if lampOffList != None:
#Retrieve the list of files to use for flat creation.
info("Attempting to combine Lamps off files in {}".format(lampOffList))
lampOffList = IO.list_file_to_strings(lampOffList)
for flat in lampOffList:
debug(str(flat))
out = os.path.join("combflat_lamps_off_2d_{:s}.fits".format(band))
IO.imcombine(flatlist, out, options, reject="minmax", nlow=1, nhigh=1)
file_on = os.path.join("combflat_2d_{:s}.fits".format(band))
file_off = os.path.join("combflat_lamps_off_2d_{:s}.fits".format(band))
file_on_save = os.path.join("combflat_lamps_on_2d_{:s}.fits".format(band))
IO.imarith(file_on, '-', file_off, file_on_save)
debug("Combined '%s' to '%s'" % (flatlist, maskname))
# info("Combined flats for '%s'" % (maskname))
path = "combflat_2d_%s.fits" % band
(header, data) = IO.readfits(path, use_bpm=True)
info("Flat written to %s" % path)
# Edge Trace
if bs.long_slit:
info( "Long slit mode recognized")
info( "Central row position: "+str(longslit["row_position"]))
info( "Upper and lower limits: "+str(longslit["yrange"][0])+" "+str(longslit["yrange"][1]))
results = find_longslit_edges(data,header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit)
elif bs.long2pos_slit:
info( "Long2pos mode recognized")
results = find_long2pos_edges(data,header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit)
else:
info('Finding slit edges in {}'.format(path))
results = find_and_fit_edges(data, header, bs, options,edgeThreshold=edgeThreshold)
results[-1]["maskname"] = maskname
results[-1]["band"] = band
np.save("slit-edges_{0}".format(band), results)
save_ds9_edges(results, options)
# Generate Flat
out = "pixelflat_2d_%s.fits" % (band)
if lampOffList != None:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=True)
else:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=False)
info( "Pixel flat took {0:6.4} s".format(time.time()-tick))
def handle_background(filelist, wavename, maskname, band_name, options,
shifts=None, plan=None, extension=None, target='default'):
'''
Perform difference imaging and subtract residual background.
The plan looks something like: [['A', 'B']]
In this case, the number of output files is equal to the length of the list (1).
If you choose to use an ABA'B' pattern then the plan will be: [["A", "B"], ["A'", "B'"]]
the background subtraction code will make and handle two files, "A-B" and "A'-B'".
'''
global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band
band = band_name
flatname = "pixelflat_2d_%s.fits" % band_name
hdr, flat = IO.readfits("pixelflat_2d_%s.fits" % (band_name), options)
if np.abs(np.median(flat) - 1) > 0.1:
error("Flat seems poorly behaved.")
raise Exception("Flat seems poorly behaved.")
'''
This next section of the code figures out the observing plan
and then deals with the bookeeping of sending the plan
to the background subtracter.
'''
hdrs = []
epss = {}
vars = {}
bss = []
times = {}
Nframes = []
i = 0
header = pf.Header()
for i in range(len(filelist)):
fl = filelist[i]
files = IO.list_file_to_strings(fl)
info("Combining observation files listed in {}".format(fl))
if shifts is None: shift = None
else: shift = shifts[i]
hdr, electron, var, bs, time, Nframe = imcombine(files, maskname,
options, flat, outname="%s.fits" % (fl),
shifts=shift, extension=extension)
hdrs.append(hdr)
header = merge_headers(header, hdr)
epss[hdr['FRAMEID']] = electron/time
vars[hdr['FRAMEID']] = var
times[hdr['FRAMEID']] = time
bss.append(bs)
Nframes.append(Nframe)
positions = {}
i = 0
for h in hdrs:
positions[h['FRAMEID']] = i
i += 1
posnames = set(positions.keys())
if plan is None:
plan = guess_plan_from_positions(posnames)
num_outputs = len(plan)
edges, meta = IO.load_edges(maskname, band, options)
lam = IO.readfits(wavename, options)
bs = bss[0]
for i in range(num_outputs):
posname0 = plan[i][0]
posname1 = plan[i][1]
info("Handling %s - %s" % (posname0, posname1))
data = epss[posname0] - epss[posname1]
Var = vars[posname0] + vars[posname1]
itime = np.mean([times[posname0], times[posname1]], axis=0)
p = Pool()
solutions = p.map(background_subtract_helper, list(range(len(bs.ssl))))
p.close()
write_outputs(solutions, itime, header, maskname, band, plan[i], options, target=target)
def quit(self, event):
info(' Done with aperture edits for this slit')
self.disconnect()
pl.close(self.fig)
def find_longslit_edges(data,
header,
bs,
options,
edgeThreshold=450,
longslit=None):
y = 2034
DY = 44.25
toc = 0
ssl = bs.ssl
slits = []
top = [0., np.float(Options.npix)]
start_slit_num = int(bs.msl[0]['Slit_Number']) - 1
if start_slit_num > 0:
y -= DY * start_slit_num
# if the mask is a long slit, the default y value will be wrong. Set instead to be the middle
if bs.long_slit:
try:
y = longslit["yrange"][1]
except:
error(
"Longslit reduction mode is specified, but the row position has not been specified. Defaulting to "
+ str(y))
print(
"Longslit reduction mode is specified, but the row position has not been specified. Defaulting to "
+ str(y))
# Count and check that the # of objects in the SSL matches that of the MSL
# This is purely a safety check
numslits = np.zeros(len(ssl))
for i in range(len(ssl)):
slit = ssl[i]
M = np.where(slit["Target_Name"] == bs.msl["Target_in_Slit"])
numslits[i] = len(M[0])
numslits = np.array(numslits)
info("Number of slits allocated for this longslit: " +
str(np.sum(numslits)))
# now begin steps outline above
results = []
result = {}
result["Target_Name"] = ssl[0]["Target_Name"]
# 1 Defines a polynomial of degree 0, which is a constant, with the value of the top of the slit
result["top"] = np.poly1d([longslit["yrange"][1]])
topfun = np.poly1d([longslit["yrange"][1]
]) # this is a constant funtion with c=top of the slit
botfun = np.poly1d([
longslit["yrange"][0]
]) # this is a constant funtion with c=bottom of the slit
# xposs_top_this = [10 110 210 .... 1810 1910]
xposs_top = np.arange(10, 2000, 100)
xposs_bot = np.arange(10, 2000, 100)
# yposs_top_this = [1104 1104 ... 1104 1104], it's the constant polynomium calculated at the X positions
yposs_top = topfun(xposs_top)
yposs_bot = botfun(xposs_bot)
''' Deal with the current slit '''
target = 0
hpps = Wavelength.estimate_half_power_points(
bs.scislit_to_csuslit(target + 1)[0], header, bs)
ok = np.where((xposs_top > hpps[0]) & (xposs_top < hpps[1]))
xposs_bot = xposs_bot[ok]
yposs_bot = yposs_bot[ok]
xposs_top = xposs_top[ok]
yposs_top = yposs_top[ok]
if len(xposs_bot) == 0:
error("The slit edges specifications appear to be incorrect.")
raise Exception(
"The slit edges specifications appear to be incorrect.")
# bot is the polynomium that defines the shape of the bottom of the slit. In this case, we set it to a constant.
bot = botfun.c.copy()
top = topfun.c.copy()
#4
result = {}
result["Target_Name"] = ssl[target]["Target_Name"]
result["xposs_top"] = xposs_top
result["yposs_top"] = yposs_top
result["xposs_bot"] = xposs_bot
result["yposs_bot"] = yposs_bot
result["top"] = np.poly1d(top)
result["bottom"] = np.poly1d(bot)
result["hpps"] = hpps
result["ok"] = ok
results.append(result)
results.append({"version": options["version"]})
return results
def delete_aperture(self, id):
pos = self.apertures[id]['position']
info(' Removing aperture at position {:.0f}'.format(pos))
self.apertures.remove_row(id)
self.plot_apertures()
def imcombine(files,
maskname,
options,
flat,
outname=None,
shifts=None,
extension=None):
'''
From a list of files it imcombine returns the imcombine of several values.
The imcombine code also estimates the readnoise ad RN/sqrt(numreads) so
that the variance per frame is equal to (ADU + RN^2) where RN is computed
in ADUs.
Arguments:
files[]: list of full path to files to combine
maskname: Name of mask
options: Options dictionary
flat[2048x2048]: Flat field (values should all be ~ 1.0)
outname: If set, will write (see notes below for details)
eps_[outname].fits: electron/sec file
itimes_[outname].fits: integration time
var_[outname].fits: Variance files
shifts[len(files)]: If set, will "roll" each file by the
amount in the shifts vector in pixels. This argument
is used when telescope tracking is poor. If you need
to use this, please notify Keck staff about poor
telescope tracking.
Returns 6-element tuple:
header: The combined header
electrons [2048x2048]: e- (in e- units)
var [2048x2048]: electrons + RN**2 (in e-^2 units)
bs: The MOSFIRE.Barset instance
itimes [2048x2048]: itimes (in s units)
Nframe: The number of frames that contribute to the summed
arrays above. If Nframe > 5 I use the sigma-clipping
Cosmic Ray Rejection tool. If Nframe < 5 then I drop
the max/min elements.
Notes:
header -- fits header
ADUs -- The mean # of ADUs per frame
var -- the Variance [in adu] per frame.
bs -- Barset
itimes -- The _total_ integration time in second
Nframe -- The number of frames in a stack.
Thus the number of electron per second is derived as:
e-/sec = (ADUs * Gain / Flat) * (Nframe/itimes)
The total number of electrons is:
el = ADUs * Gain * Nframe
'''
ADUs = np.zeros((len(files), 2048, 2048))
itimes = np.zeros((len(files), 2048, 2048))
prevssl = None
prevmn = None
patternid = None
maskname = None
header = None
if shifts is None:
shifts = np.zeros(len(files))
warnings.filterwarnings('ignore')
for i in range(len(files)):
fname = files[i]
thishdr, data, bs = IO.readmosfits(fname, options, extension=extension)
itimes[i, :, :] = thishdr["truitime"]
base = os.path.basename(fname).rstrip(".fits")
fnum = int(base.split("_")[1])
if shifts[i] == 0:
ADUs[i, :, :] = data.filled(0.0) / flat
else:
ADUs[i, :, :] = np.roll(data.filled(0.0) / flat,
np.int(shifts[i]),
axis=0)
''' Construct Header'''
if header is None:
header = thishdr
header["imfno%3.3i" % (fnum)] = (fname, "img%3.3i file name" % fnum)
list(
map(lambda x: rem_header_key(header, x), [
"CTYPE1", "CTYPE2", "WCSDIM", "CD1_1", "CD1_2", "CD2_1",
"CD2_2", "LTM1_1", "LTM2_2", "WAT0_001", "WAT1_001",
"WAT2_001", "CRVAL1", "CRVAL2", "CRPIX1", "CRPIX2", "RADECSYS"
]))
for card in header.cards:
if card == '': continue
key, val, comment = card
if key in thishdr:
if val != thishdr[key]:
newkey = key + ("_img%2.2i" % fnum)
try:
header[newkey.rstrip()] = (thishdr[key], comment)
except:
pass
''' Now handle error checking'''
if maskname is not None:
if thishdr["maskname"] != maskname:
error("File %s uses mask '%s' but the stack is of '%s'" %
(fname, thishdr["maskname"], maskname))
raise Exception(
"File %s uses mask '%s' but the stack is of '%s'" %
(fname, thishdr["maskname"], maskname))
maskname = thishdr["maskname"]
if thishdr["aborted"]:
error("Img '%s' was aborted and should not be used" % fname)
raise Exception("Img '%s' was aborted and should not be used" %
fname)
if prevssl is not None:
if len(prevssl) != len(bs.ssl):
# todo Improve these checks
error("The stack of input files seems to be of "
"different masks")
raise Exception("The stack of input files seems to be of "
"different masks")
prevssl = bs.ssl
if patternid is not None:
if patternid != thishdr["frameid"]:
error("The stack should be of '%s' frames only, but "
"the current image is a '%s' frame." %
(patternid, thishdr["frameid"]))
raise Exception("The stack should be of '%s' frames only, but "
"the current image is a '%s' frame." %
(patternid, thishdr["frameid"]))
patternid = thishdr["frameid"]
if maskname is not None:
if maskname != thishdr["maskname"]:
error("The stack should be of CSU mask '%s' frames "
"only but contains a frame of '%s'." %
(maskname, thishdr["maskname"]))
raise Exception("The stack should be of CSU mask '%s' frames "
"only but contains a frame of '%s'." %
(maskname, thishdr["maskname"]))
maskname = thishdr["maskname"]
if thishdr["BUNIT"] != "ADU per coadd":
error("The units of '%s' are not in ADU per coadd and "
"this violates an assumption of the DRP. Some new code "
"is needed in the DRP to handle the new units of "
"'%s'." % (fname, thishdr["BUNIT"]))
raise Exception(
"The units of '%s' are not in ADU per coadd and "
"this violates an assumption of the DRP. Some new code "
"is needed in the DRP to handle the new units of "
"'%s'." % (fname, thishdr["BUNIT"]))
''' Error checking is complete'''
debug("%s %s[%s]/%s: %5.1f s, Shift: %i px" %
(fname, maskname, patternid, header['filter'], np.mean(
itimes[i]), shifts[i]))
warnings.filterwarnings('always')
# the electrons and el_per_sec arrays are:
# [2048, 2048, len(files)] and contain values for
# each individual frame that is being combined.
# These need to be kept here for CRR reasons.
electrons = np.array(ADUs) * Detector.gain
el_per_sec = electrons / itimes
output = np.zeros((2048, 2048))
exptime = np.zeros((2048, 2048))
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
# Cosmic ray rejection code begins here. This code construction the
# electrons and itimes arrays.
standard = True
new_from_chuck = False
# Chuck Steidel has provided a modified version of the CRR procedure.
# to enable it, modify the variables above.
if new_from_chuck and not standard:
if len(files) >= 5:
print("Sigclip CRR")
srt = np.argsort(electrons, axis=0, kind='quicksort')
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
el_per_sec = el_per_sec[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
# Construct the mean and standard deviation by dropping the top and bottom two
# electron fluxes. This is temporary.
mean = np.mean(el_per_sec[1:-1, :, :], axis=0)
std = np.std(el_per_sec[1:-1, :, :], axis=0)
drop = np.where((el_per_sec > (mean + std * 4))
| (el_per_sec < (mean - std * 4)))
print("dropping: ", len(drop[0]))
electrons[drop] = 0.0
itimes[drop] = 0.0
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
else:
warning("With less than 5 frames, the pipeline does NOT perform")
warning("Cosmic Ray Rejection.")
# the "if false" line disables cosmic ray rejection"
if False:
for i in range(len(files)):
el = electrons[i, :, :]
it = itimes[i, :, :]
el_mf = scipy.signal.medfilt(el, 5)
bad = np.abs(el - el_mf) / np.abs(el) > 10.0
el[bad] = 0.0
it[bad] = 0.0
electrons[i, :, :] = el
itimes[i, :, :] = it
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
if standard and not new_from_chuck:
if len(files) >= 9:
info("Sigclip CRR")
srt = np.argsort(electrons, axis=0, kind='quicksort')
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
el_per_sec = el_per_sec[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
# Construct the mean and standard deviation by dropping the top and bottom two
# electron fluxes. This is temporary.
mean = np.mean(el_per_sec[2:-2, :, :], axis=0)
std = np.std(el_per_sec[2:-2, :, :], axis=0)
drop = np.where((el_per_sec > (mean + std * 4))
| (el_per_sec < (mean - std * 4)))
info("dropping: " + str(len(drop[0])))
electrons[drop] = 0.0
itimes[drop] = 0.0
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
elif len(files) > 5:
warning("WARNING: Drop min/max CRR")
srt = np.argsort(el_per_sec, axis=0)
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
electrons = np.sum(electrons[1:-1, :, :], axis=0)
itimes = np.sum(itimes[1:-1, :, :], axis=0)
Nframe = len(files) - 2
else:
warning("With less than 5 frames, the pipeline does NOT perform")
warning("Cosmic Ray Rejection.")
# the "if false" line disables cosmic ray rejection"
if False:
for i in range(len(files)):
el = electrons[i, :, :]
it = itimes[i, :, :]
# calculate the median image
el_mf = scipy.signal.medfilt(el, 5)
el_mf_large = scipy.signal.medfilt(el_mf, 15)
# LR: this is a modified version I was experimenting with. For the version
# written by Nick, see the new_from_chuck part of this code
# sky sub
el_sky_sub = el_mf - el_mf_large
# add a constant value
el_plus_constant = el_sky_sub + 100
bad = np.abs(el - el_mf) / np.abs(el_plus_constant) > 50.0
el[bad] = 0.0
it[bad] = 0.0
electrons[i, :, :] = el
itimes[i, :, :] = it
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
''' Now handle variance '''
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
var = (electrons + RN**2)
''' Now mask out bad pixels '''
electrons[data.mask] = np.nan
var[data.mask] = np.inf
if "RN" in header:
error("RN Already populated in header")
raise Exception("RN Already populated in header")
header['RN'] = ("%1.3f", "Read noise in e-")
header['NUMFRM'] = (Nframe, 'Typical number of frames in stack')
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(np.float32(electrons / itimes),
maskname,
"eps_%s" % (outname),
options,
header=header,
overwrite=True)
# Update itimes after division in order to not introduce nans
itimes[data.mask] = 0.0
header['BUNIT'] = 'ELECTRONS^2'
IO.writefits(var,
maskname,
"var_%s" % (outname),
options,
header=header,
overwrite=True,
lossy_compress=True)
header['BUNIT'] = 'SECOND'
IO.writefits(np.float32(itimes),
maskname,
"itimes_%s" % (outname),
options,
header=header,
overwrite=True,
lossy_compress=True)
return header, electrons, var, bs, itimes, Nframe
def background_subtract_helper(slitno):
'''
Background subtraction follows the methods outlined by Kelson (2003). Here
a background is estimated as a function of wavelength using B-splines and
subtracted off. The assumption is that background is primarily a function
of wavelength, and thus by sampling the background across the full 2-d
spectrum the background is sampled at much higher than the native spectral
resolution of mosfire.
Formally, the assumption that background is only a function of wavelength
is incorrect, and indeed a "transmission function" is estimated from the 2d
spectrum. This gives an estimate of the throughput of the slit and divided
out.
1. Extract the slit from the 2d image.
2. Convert the 2d spectrum into a 1d spectrum
3. Estimate transmission function
'''
global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band
tick = time.time()
# 1
top = np.int(edges[slitno]["top"](1024))
bottom = np.int(edges[slitno]["bottom"](1024))
info("Background subtracting slit %i [%i,%i]" % (slitno, top, bottom))
pix = np.arange(2048)
xroi = slice(0,2048)
yroi = slice(bottom, top)
stime = itime[yroi, xroi]
slit = data[yroi, xroi]
Var[np.logical_not(np.isfinite(Var))] = np.inf
lslit = lam[1][yroi,xroi]
# 2
xx = np.arange(slit.shape[1])
yy = np.arange(slit.shape[0])
X,Y = np.meshgrid(xx,yy)
train_roi = slice(5,-5)
ls = lslit[train_roi].flatten().filled(0)
ss = slit[train_roi].flatten()
ys = Y[train_roi].flatten()
dl = np.ma.median(np.diff(lslit[lslit.shape[0]//2,:]))
if dl == 0:
return {"ok": False}
sort = np.argsort(ls)
ls = ls[sort]
ys = ys[sort]
hpps = np.array(Filters.hpp[band])
diff = np.append(np.diff(ls), False)
OK = (diff > 0.001) & (ls > hpps[0]) & (ls < hpps[1]) & (np.isfinite(ls)) \
& (np.isfinite(ss[sort]))
if len(np.where(OK)[0]) < 1000:
warning("Failed on slit "+str(slitno))
return {"ok": False}
# 3
pp = np.poly1d([1.0])
ss = (slit[train_roi] / pp(Y[train_roi])).flatten()
ss = ss[sort]
knotstart = max(hpps[0], min(ls[OK])) + 5
knotend = min(hpps[1], max(ls[OK])) - 5
for i in range(3):
try:
delta = dl*0.9
knots = np.arange(knotstart, knotend, delta)
bspline = II.splrep(ls[OK], ss[OK], k=5, task=-1, t=knots)
except ValueError as e:
warning('Failed to fit spline with delta = {:5f}'.format(delta))
warning(str(e))
delta = dl*1.4
info('Trying with delta = {:5f}'.format(delta))
knots = np.arange(knotstart, knotend, delta)
try:
bspline = II.splrep(ls[OK], ss[OK], k=5, task=-1, t=knots)
except ValueError as e:
warning("Could not construct spline on slit "+str(slitno))
warning(str(e))
return {"ok": False}
ll = lslit.flatten()
model = II.splev(ll, bspline)
oob = np.where((ll < knotstart) | (ll > knotend))
model[oob] = np.median(ss[~np.isnan(ss)])
model = model.reshape(slit.shape)
output = slit - model
std = np.abs(output)/(np.sqrt(np.abs(model)+1))
tOK = (std[train_roi] < 10).flatten() & \
np.isfinite(std[train_roi]).flatten()
OK = OK & tOK[sort]
return {"ok": True, "slitno": slitno, "bottom": bottom, "top": top,
"output": output, "model": model, "bspline": bspline}
## Output the file list to a text file for later examination
if os.path.exists('filelist.txt'):
debug('Removing old filelist.txt')
os.remove('filelist.txt')
fl = open('filelist.txt', 'w')
files = []
for i in range(1, len(sys.argv)):
files.extend(glob.iglob(os.path.abspath(sys.argv[i])))
files = [file for file in files if os.path.splitext(file)[1] != '.original']
masks = {}
info('Examining {} files'.format(len(files)))
for fname in files:
try:
header = IO.readheader(fname)
except IOError: #, err:
fl.write("Couldn't IO %s\n" % fname)
continue
except:
fl.write("%s is unreadable\n" % fname)
continue
lamps = ""
try:
if header["pwstata7"] == 1:
lamps += header["pwloca7"][0:2]
## Output the file list to a text file for later examination
if os.path.exists('filelist.txt'):
debug('Removing old filelist.txt')
os.remove('filelist.txt')
fl = open('filelist.txt', 'w')
files = []
for i in range(1, len(sys.argv)):
files.extend(glob.iglob(sys.argv[i]))
masks = {}
info('Examining {} files'.format(len(files)))
for fname in files:
try:
header = IO.readheader(fname)
except IOError:#, err:
fl.write("Couldn't IO %s\n" % fname)
continue
except:
fl.write("%s is unreadable\n" % fname)
continue
lamps = ""
try:
if header["pwstata7"] == 1:
lamps += header["pwloca7"][0:2]
def handle_rectification_helper(edgeno):
''' All the rectification happens in this helper function. This helper function
is spawned as a separate process in the multiprocessing pool'''
global edges, dats, vars, itimes, shifts, lambdas, band, fidl,all_shifts
pix = np.arange(2048)
edge = edges[edgeno]
info("Handling edge: "+str(edge["Target_Name"]))
tops = edge["top"](pix)
bots = edge["bottom"](pix)
# Length of the slit in arcsecond
lenas = (tops[1024] - bots[1024]) * 0.18
mxshift = np.abs(np.int(np.ceil(np.max(all_shifts)/0.18)))
mnshift = np.abs(np.int(np.floor(np.min(all_shifts)/0.18)))
top = int(min(np.floor(np.min(tops)), 2048))
bot = int(max(np.ceil(np.max(bots)), 0))
ll = lambdas[1].data[bot:top, :]
eps = dats[1][bot:top, :].filled(0.0)
vv = vars[1][bot:top, :].filled(np.inf)
it = itimes[1][bot:top, :].filled(0.0)
lmid = ll[ll.shape[0]//2,:]
hpp = Filters.hpp[band]
minl = lmid[0] if lmid[0]>hpp[0] else hpp[0]
maxl = lmid[-1] if lmid[-1]<hpp[1] else hpp[1]
epss = []
ivss = []
itss = []
if len(shifts) is 1: sign = 1
else: sign = -1
for shift in shifts:
output = r_interpol(ll, eps, fidl, tops, top, shift_pix=shift/0.18,
pad=[mnshift, mxshift], fill_value = np.nan)
epss.append(sign * output)
ivar = 1/vv
bad = np.where(np.isfinite(ivar) ==0)
ivar[bad] = 0.0
output = r_interpol(ll, ivar, fidl, tops, top, shift_pix=shift/0.18,
pad=[mnshift, mxshift], fill_value=np.nan)
ivss.append(output)
output = r_interpol(ll, it, fidl, tops, top, shift_pix=shift/0.18,
pad=[mnshift, mxshift], fill_value=np.nan)
itss.append(output)
sign *= -1
# the "mean of empty slice" warning are generated at the top and bottom edges of the array
# where there is basically no data due to the shifts between a and b positions
# we could pad a little bit less, or accept the fact that the slits have a couple of rows of
# nans in the results.
warnings.filterwarnings('ignore','Mean of empty slice')
it_img = np.nansum(np.array(itss), axis=0)
eps_img = np.nanmean(epss, axis=0)
warnings.filterwarnings('always')
# Remove any NaNs or infs from the variance array
ivar_img = []
for ivar in ivss:
bad = np.where(np.isfinite(ivar) == 0)
ivar[bad] = 0.0
ivar_img.append(ivar)
IV = np.array(ivar_img)
bad = np.isclose(IV,0)
IV[bad] = np.inf
var_img = np.nanmean(1/np.array(IV), axis=0)
sd_img = np.sqrt(var_img)
return {"eps_img": eps_img, "sd_img": sd_img, "itime_img": it_img,
"lambda": fidl, "Target_Name": edge["Target_Name"],
"slitno": edgeno+1, "offset": np.max(tops-top)}
def make_pixel_flat(data, results, options, outfile, inputs, lampsOff=None):
'''
Convert a flat image into a flat field
'''
def pixel_min(y):
return int(np.floor(np.min(y)))
def pixel_max(y):
return int(np.ceil(np.max(y)))
def collapse_flat_box(dat):
'''Collapse data to the spectral axis (0)'''
v = np.median(dat, axis=0).ravel()
return v
flat = np.ones(shape=Detector.npix)
hdu = pf.PrimaryHDU((data / flat).astype(np.float32))
hdu.header.set("version", __version__, "DRP version")
i = 0
for flatname in inputs:
nm = flatname.split("/")[-1]
hdu.header.set("infile%2.2i" % i, nm)
i += 1
slitno = 0
for result in results[0:-1]:
slitno += 1
#There seems to be a bit of an issue with this on Longslits, where it is being read as bites not as str
try:
hdu.header.set("targ%2.2i" % slitno, result["Target_Name"])
except ValueError:
hdu.header.set("targ%2.2i" % slitno,
str(result["Target_Name"], 'utf-8'))
bf = result["bottom"]
tf = result["top"]
try:
hpps = result["hpps"]
except:
error("No half power points for this slit")
hpps = [0, Detector.npix[0]]
xs = np.arange(hpps[0], hpps[1])
top = pixel_min(tf(xs))
bottom = pixel_max(bf(xs))
hdu.header.set("top%2.2i" % slitno, top)
hdu.header.set("bottom%2.2i" % slitno, bottom)
info("%s] Bounding top/bottom: %i/%i" %
(result["Target_Name"], bottom, top))
v = collapse_flat_box(data[bottom:top, hpps[0]:hpps[1]])
x2048 = np.arange(Options.npix)
v = np.poly1d(np.polyfit(xs, v,
options['flat-field-order']))(xs).ravel()
for i in np.arange(bottom - 1, top):
flat[i, hpps[0]:hpps[1]] = v
info("Producing Pixel Flat...")
for r in range(len(results) - 1):
theslit = results[r]
try:
bf = theslit["bottom"]
tf = theslit["top"]
except:
pdb.set_trace()
for i in range(hpps[0], hpps[1]):
top = int(np.floor(tf(i)))
bottom = int(np.ceil(bf(i)))
data[top:bottom, i] = flat[top:bottom, i]
hdu.data = (data / flat).astype(np.float32)
bad = np.abs(hdu.data - 1.0) > 0.5
hdu.data[bad] = 1.0
hdu.data = hdu.data.filled(1)
if os.path.exists(outfile):
os.remove(outfile)
hdu.writeto(outfile)
info("Done.")
def background_subtract_helper(slitno):
'''
Background subtraction follows the methods outlined by Kelson (2003). Here
a background is estimated as a function of wavelength using B-splines and
subtracted off. The assumption is that background is primarily a function
of wavelength, and thus by sampling the background across the full 2-d
spectrum the background is sampled at much higher than the native spectral
resolution of mosfire.
Formally, the assumption that background is only a function of wavelength
is incorrect, and indeed a "transmission function" is estimated from the 2d
spectrum. This gives an estimate of the throughput of the slit and divided
out.
1. Extract the slit from the 2d image.
2. Convert the 2d spectrum into a 1d spectrum
3. Estimate transmission function
'''
global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band
tick = time.time()
# 1
top = np.int(edges[slitno]["top"](1024))
bottom = np.int(edges[slitno]["bottom"](1024))
info("Background subtracting slit %i [%i,%i]" % (slitno, top, bottom))
pix = np.arange(2048)
xroi = slice(0, 2048)
yroi = slice(bottom, top)
stime = itime[yroi, xroi]
slit = data[yroi, xroi]
Var[np.logical_not(np.isfinite(Var))] = np.inf
lslit = lam[1][yroi, xroi]
# 2
xx = np.arange(slit.shape[1])
yy = np.arange(slit.shape[0])
X, Y = np.meshgrid(xx, yy)
train_roi = slice(5, -5)
ls = lslit[train_roi].flatten().filled(0)
ss = slit[train_roi].flatten()
ys = Y[train_roi].flatten()
dl = np.ma.median(np.diff(lslit[lslit.shape[0] // 2, :]))
if dl == 0:
return {"ok": False}
sort = np.argsort(ls)
ls = ls[sort]
ys = ys[sort]
hpps = np.array(Filters.hpp[band])
diff = np.append(np.diff(ls), False)
OK = (diff > 0.001) & (ls > hpps[0]) & (ls < hpps[1]) & (np.isfinite(ls)) \
& (np.isfinite(ss[sort]))
if len(np.where(OK)[0]) < 1000:
warning("Failed on slit " + str(slitno))
return {"ok": False}
# 3
pp = np.poly1d([1.0])
ss = (slit[train_roi] / pp(Y[train_roi])).flatten()
ss = ss[sort]
knotstart = max(hpps[0], min(ls[OK])) + 5
knotend = min(hpps[1], max(ls[OK])) - 5
for i in range(3):
try:
delta = dl * 0.9
knots = np.arange(knotstart, knotend, delta)
bspline = II.splrep(ls[OK], ss[OK], k=5, task=-1, t=knots)
except ValueError as e:
warning('Failed to fit spline with delta = {:5f}'.format(delta))
warning(str(e))
delta = dl * 1.4
info('Trying with delta = {:5f}'.format(delta))
knots = np.arange(knotstart, knotend, delta)
try:
bspline = II.splrep(ls[OK], ss[OK], k=5, task=-1, t=knots)
except ValueError as e:
warning("Could not construct spline on slit " + str(slitno))
warning(str(e))
return {"ok": False}
ll = lslit.flatten()
model = II.splev(ll, bspline)
oob = np.where((ll < knotstart) | (ll > knotend))
model[oob] = np.median(ss[~np.isnan(ss)])
model = model.reshape(slit.shape)
output = slit - model
std = np.abs(output) / (np.sqrt(np.abs(model) + 1))
tOK = (std[train_roi] < 10).flatten() & \
np.isfinite(std[train_roi]).flatten()
OK = OK & tOK[sort]
return {
"ok": True,
"slitno": slitno,
"bottom": bottom,
"top": top,
"output": output,
"model": model,
"bspline": bspline
}
def find_edge_pair(data, y, roi_width, edgeThreshold=450):
'''
find_edge_pair finds the edge of a slit pair in a flat
data[2048x2048]: a well illuminated flat field [DN]
y: guess of slit edge position [pix]
Keywords:
edgeThreshold: the pixel value below which we should ignore using
to calculate edges.
Moves along the edge of a slit image
- At each location along the slit edge, determines
the position of the demarcations between two slits
Outputs:
xposs []: Array of x positions along the slit edge [pix]
yposs []: The fitted y positions of the "top" edge of the slit [pix]
widths []: The fitted delta from the top edge of the bottom [pix]
scatters []: The amount of light between slits
The procedure is as follows
1: starting from a guess spatial position (parameter y), march
along the spectral direction in some chunk of pixels
2: At each spectral location, construct a cross cut across the
spatial direction; select_roi is used for this.
3: Fit a two-sided error function Fit.residual_disjoint_pair
on the vertical cross cut derived in step 2.
4: If the fit fails, store it in the missing list
- else if the top fit is good, store the top values in top vector
- else if the bottom fit is good, store the bottom values in bottom
vector.
5: In the vertical cross-cut, there is a minimum value. This minimum
value is stored as a measure of scattered light.
Another procedure is used to fit polynomials to these fitted values.
'''
def select_roi(data, roi_width):
v = data[int(y - roi_width):int(y + roi_width),
int(xp) - 2:int(xp) + 2]
v = np.median(v, axis=1) # Axis = 1 is spatial direction
return v
xposs_top = []
yposs_top = []
xposs_top_missing = []
xposs_bot = []
yposs_bot = []
xposs_bot_missing = []
yposs_bot_scatters = []
#1
rng = np.linspace(10, 2040, 50).astype(np.int)
for i in rng:
xp = i
#2
v = select_roi(data, roi_width)
xs = np.arange(len(v))
# Modified from 450 as the hard coded threshold to one that
# can be controlled by a keyword
if (np.median(v) < edgeThreshold):
xposs_top_missing.append(xp)
xposs_bot_missing.append(xp)
continue
#3
ff = Fit.do_fit(v, residual_fun=Fit.residual_disjoint_pair)
fit_ok = 0 < ff[4] < 4
if fit_ok:
(sigma, offset, mult1, mult2, add, width) = ff[0]
xposs_top.append(xp)
yposs_top.append(y - roi_width + offset + width)
xposs_bot.append(xp)
yposs_bot.append(y - roi_width + offset)
between = offset + width // 2
if 0 < between < len(v) - 1:
start = int(np.max([0, between - 2]))
stop = int(np.min([len(v), between + 2]))
yposs_bot_scatters.append(np.min(v[start:stop])) # 5
if False:
pl.figure(2)
pl.clf()
tmppix = np.arange(y - roi_width, y + roi_width)
tmpx = np.arange(len(v))
pl.axvline(y - roi_width + offset + width, color='red')
pl.axvline(y - roi_width + offset, color='red')
pl.scatter(tmppix, v)
pl.plot(tmppix, Fit.fit_disjoint_pair(ff[0], tmpx))
pl.axhline(yposs_bot_scatters[-1])
pl.draw()
else:
yposs_bot_scatters.append(np.nan)
else:
xposs_bot_missing.append(xp)
xposs_top_missing.append(xp)
info("Skipping wavelength pixel): %i" % (xp))
return list(
map(np.array, (xposs_bot, xposs_bot_missing, yposs_bot, xposs_top,
xposs_top_missing, yposs_top, yposs_bot_scatters)))
def make_pixel_flat(data, results, options, outfile, inputs, lampsOff=None):
'''
Convert a flat image into a flat field
'''
def pixel_min(y): return int(np.floor(np.min(y)))
def pixel_max(y): return int(np.ceil(np.max(y)))
def collapse_flat_box(dat):
'''Collapse data to the spectral axis (0)'''
v = np.median(dat, axis=0).ravel()
return v
flat = np.ones(shape=Detector.npix)
hdu = pf.PrimaryHDU((data/flat).astype(np.float32))
hdu.header.set("version", __version__, "DRP version")
i = 0
for flatname in inputs:
nm = flatname.split("/")[-1]
hdu.header.set("infile%2.2i" % i, nm)
i += 1
slitno = 0
for result in results[0:-1]:
slitno += 1
#There seems to be a bit of an issue with this on Longslits, where it is being read as bites not as str
try:
hdu.header.set("targ%2.2i" % slitno, result["Target_Name"])
except ValueError: hdu.header.set("targ%2.2i" % slitno, str(result["Target_Name"], 'utf-8'))
bf = result["bottom"]
tf = result["top"]
try:
hpps = result["hpps"]
except:
error( "No half power points for this slit")
hpps = [0, Detector.npix[0]]
xs = np.arange(hpps[0], hpps[1])
top = pixel_min(tf(xs))
bottom = pixel_max(bf(xs))
hdu.header.set("top%2.2i" % slitno, top)
hdu.header.set("bottom%2.2i" % slitno, bottom)
info( "%s] Bounding top/bottom: %i/%i" % (result["Target_Name"],
bottom, top))
v = collapse_flat_box(data[bottom:top,hpps[0]:hpps[1]])
x2048 = np.arange(Options.npix)
v = np.poly1d(np.polyfit(xs,v,
options['flat-field-order']))(xs).ravel()
for i in np.arange(bottom-1, top):
flat[i,hpps[0]:hpps[1]] = v
info("Producing Pixel Flat...")
for r in range(len(results)-1):
theslit = results[r]
try:
bf = theslit["bottom"]
tf = theslit["top"]
except:
pdb.set_trace()
for i in range(hpps[0], hpps[1]):
top = int(np.floor(tf(i)))
bottom = int(np.ceil(bf(i)))
data[top:bottom, i] = flat[top:bottom,i]
hdu.data = (data/flat).astype(np.float32)
bad = np.abs(hdu.data-1.0) > 0.5
hdu.data[bad] = 1.0
hdu.data = hdu.data.filled(1)
if os.path.exists(outfile):
os.remove(outfile)
hdu.writeto(outfile)
info("Done.")
def handle_flats(flatlist,
maskname,
band,
options,
extension=None,
edgeThreshold=450,
lampOffList=None,
longslit=None):
'''
handle_flats is the primary entry point to the Flats module.
handle_flats takes a list of individual exposure FITS files and creates:
1. A CRR, dark subtracted, pixel-response flat file.
2. A set of polynomials that mark the edges of a slit
Inputs:
flatlist:
maskname: The name of a mask
band: A string indicating the bandceil
Outputs:
file {maskname}/flat_2d_{band}.fits -- pixel response flat
file {maskname}/edges.np
'''
tick = time.time()
# Check
bpos = np.ones(92) * -1
#Retrieve the list of files to use for flat creation.
flatlist = IO.list_file_to_strings(flatlist)
if len(flatlist) == 0:
print('WARNING: No flat files found.')
raise IOError('No flat files found')
# Print the filenames to Standard-out
for flat in flatlist:
debug(str(flat))
#Determine if flat files headers are in agreement
for fname in flatlist:
hdr, dat, bs = IO.readmosfits(fname, options, extension=extension)
try:
bs0
except:
bs0 = bs
if np.any(bs0.pos != bs.pos):
print("bs0: " + str(bs0.pos) + " bs: " + str(bs.pos))
error("Barset do not seem to match")
raise Exception("Barsets do not seem to match")
if hdr["filter"] != band:
error("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
raise Exception("Filter name %s does not match header filter name "
"%s in file %s" % (band, hdr["filter"], fname))
for i in range(len(bpos)):
b = hdr["B{0:02d}POS".format(i + 1)]
if bpos[i] == -1:
bpos[i] = b
else:
if bpos[i] != b:
error("Bar positions are not all the same in "
"this set of flat files")
raise Exception("Bar positions are not all the same in "
"this set of flat files")
bs = bs0
# Imcombine the lamps ON flats
info("Attempting to combine files in {}".format(flatlist))
out = os.path.join("combflat_2d_{:s}.fits".format(band))
IO.imcombine(flatlist, out, options, reject="minmax", nlow=1, nhigh=1)
# Imcombine the lamps OFF flats and subtract the off from the On sets
if lampOffList != None:
#Retrieve the list of files to use for flat creation.
info("*********** Attempting to combine Lamps off files in {}".format(
lampOffList))
lampOffList = IO.list_file_to_strings(lampOffList)
for flat in lampOffList:
debug(str(flat))
out = os.path.join("combflat_lamps_off_2d_{:s}.fits".format(band))
IO.imcombine(lampOffList,
out,
options,
reject="minmax",
nlow=1,
nhigh=1)
file_on = os.path.join("combflat_2d_{:s}.fits".format(band))
file_off = os.path.join("combflat_lamps_off_2d_{:s}.fits".format(band))
file_on_save = os.path.join(
"combflat_lamps_on_2d_{:s}.fits".format(band))
IO.imarith(file_on, '-', file_off, file_on_save)
debug("Combined '%s' to '%s'" % (flatlist, maskname))
# info("Combined flats for '%s'" % (maskname))
path = "combflat_2d_%s.fits" % band
if lampOffList != None:
debug("Using on-off flat for K band")
path = os.path.join("combflat_lamps_on_2d_{:s}.fits".format(band))
(header, data) = IO.readfits(path, use_bpm=True)
info("Flat written to %s" % path)
# Edge Trace
if bs.long_slit:
info("Long slit mode recognized")
info("Central row position: " + str(longslit["row_position"]))
info("Upper and lower limits: " + str(longslit["yrange"][0]) + " " +
str(longslit["yrange"][1]))
results = find_longslit_edges(data,
header,
bs,
options,
edgeThreshold=edgeThreshold,
longslit=longslit)
elif bs.long2pos_slit:
info("Long2pos mode recognized")
results = find_long2pos_edges(data,
header,
bs,
options,
edgeThreshold=edgeThreshold,
longslit=longslit)
else:
info('Finding slit edges in {}'.format(path))
results = find_and_fit_edges(data,
header,
bs,
options,
edgeThreshold=edgeThreshold)
results[-1]["maskname"] = maskname
results[-1]["band"] = band
np.save("slit-edges_{0}".format(band), results)
save_ds9_edges(results, options)
# Generate Flat
out = "pixelflat_2d_%s.fits" % (band)
if lampOffList != None:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=True)
else:
make_pixel_flat(data, results, options, out, flatlist, lampsOff=False)
info("Pixel flat took {0:6.4} s".format(time.time() - tick))
def find_edge_pair(data, y, roi_width, edgeThreshold=450):
'''
find_edge_pair finds the edge of a slit pair in a flat
data[2048x2048]: a well illuminated flat field [DN]
y: guess of slit edge position [pix]
Keywords:
edgeThreshold: the pixel value below which we should ignore using
to calculate edges.
Moves along the edge of a slit image
- At each location along the slit edge, determines
the position of the demarcations between two slits
Outputs:
xposs []: Array of x positions along the slit edge [pix]
yposs []: The fitted y positions of the "top" edge of the slit [pix]
widths []: The fitted delta from the top edge of the bottom [pix]
scatters []: The amount of light between slits
The procedure is as follows
1: starting from a guess spatial position (parameter y), march
along the spectral direction in some chunk of pixels
2: At each spectral location, construct a cross cut across the
spatial direction; select_roi is used for this.
3: Fit a two-sided error function Fit.residual_disjoint_pair
on the vertical cross cut derived in step 2.
4: If the fit fails, store it in the missing list
- else if the top fit is good, store the top values in top vector
- else if the bottom fit is good, store the bottom values in bottom
vector.
5: In the vertical cross-cut, there is a minimum value. This minimum
value is stored as a measure of scattered light.
Another procedure is used to fit polynomials to these fitted values.
'''
def select_roi(data, roi_width):
v = data[int(y-roi_width):int(y+roi_width), int(xp)-2:int(xp)+2]
v = np.median(v, axis=1) # Axis = 1 is spatial direction
return v
xposs_top = []
yposs_top = []
xposs_top_missing = []
xposs_bot = []
yposs_bot = []
xposs_bot_missing = []
yposs_bot_scatters = []
#1
rng = np.linspace(10, 2040, 50).astype(np.int)
for i in rng:
xp = i
#2
v = select_roi(data, roi_width)
xs = np.arange(len(v))
# Modified from 450 as the hard coded threshold to one that
# can be controlled by a keyword
if (np.median(v) < edgeThreshold):
xposs_top_missing.append(xp)
xposs_bot_missing.append(xp)
continue
#3
ff = Fit.do_fit(v, residual_fun=Fit.residual_disjoint_pair)
fit_ok = 0 < ff[4] < 4
if fit_ok:
(sigma, offset, mult1, mult2, add, width) = ff[0]
xposs_top.append(xp)
yposs_top.append(y - roi_width + offset + width)
xposs_bot.append(xp)
yposs_bot.append(y - roi_width + offset)
between = offset + width//2
if 0 < between < len(v)-1:
start = int(np.max([0, between-2]))
stop = int(np.min([len(v),between+2]))
yposs_bot_scatters.append(np.min(v[start:stop])) # 5
if False:
pl.figure(2)
pl.clf()
tmppix = np.arange(y-roi_width, y+roi_width)
tmpx = np.arange(len(v))
pl.axvline(y - roi_width + offset + width, color='red')
pl.axvline(y - roi_width + offset, color='red')
pl.scatter(tmppix, v)
pl.plot(tmppix, Fit.fit_disjoint_pair(ff[0], tmpx))
pl.axhline(yposs_bot_scatters[-1])
pl.draw()
else:
yposs_bot_scatters.append(np.nan)
else:
xposs_bot_missing.append(xp)
xposs_top_missing.append(xp)
info("Skipping wavelength pixel): %i" % (xp))
return list(map(np.array, (xposs_bot, xposs_bot_missing, yposs_bot, xposs_top,
xposs_top_missing, yposs_top, yposs_bot_scatters)))
def find_and_fit_edges(data, header, bs, options, edgeThreshold=450):
'''
Given a flat field image, find_and_fit_edges determines the position
of all slits.
The function works by starting with a guess at the location for a slit
edge in the spatial direction(options["first-slit-edge"]).
Starting from the guess, find_edge_pair works out in either direction,
measuring the position of the (e.g.) bottom of slit 1 and top of slit 2:
------ pixel y value = 2048
Slit 1 data
------ (bottom)
deadband
------ (top)
Slit N pixel data ....
------- (bottom) pixel = 0
--------------------------------> Spectral direction
1. At the top of the flat, the slit edge is defined to be a pixel value
2. The code guesses the position of the bottom of the slit, and runs
find_edge_pair to measure slit edge locations.
3. A low-order polynomial is fit to the edge locations with
fit_edge_poly
4. The top and bottom of the current slit, is stored into the
result list.
5. The top of the next slit is stored temporarily for the next
iteration of the for loop.
6. At the bottom of the flat, the slit edge is defined to be pixel 4.
options:
options["edge-order"] -- The order of the polynomial [pixels] edge.
options["edge-fit-width"] -- The length [pixels] of the edge to
fit over
'''
# TODO: move hardcoded values into Options.py
# y is the location to start
y = 2034
DY = 44.25
toc = 0
ssl = bs.ssl
slits = []
top = [0., np.float(Options.npix)]
start_slit_num = int(bs.msl[0]['Slit_Number']) - 1
if start_slit_num > 0:
y -= DY * start_slit_num
# Count and check that the # of objects in the SSL matches that of the MSL
# This is purely a safety check
numslits = np.zeros(len(ssl))
for i in range(len(ssl)):
slit = ssl[i]
M = np.where(slit["Target_Name"] == bs.msl["Target_in_Slit"])
numslits[i] = len(M[0])
numslits = np.array(numslits)
if (np.sum(numslits) !=
CSU.numslits) and (not bs.long_slit) and (not bs.long2pos_slit):
error("The number of allocated CSU slits (%i) does not match "
" the number of possible slits (%i)." %
(np.sum(numslits), CSU.numslits))
raise Exception(
"The number of allocated CSU slits (%i) does not match "
" the number of possible slits (%i)." %
(np.sum(numslits), CSU.numslits))
# if the mask is a long slit, the default y value will be wrong. Set instead to be the middle
if bs.long_slit:
y = 1104
# now begin steps outline above
results = []
result = {}
result["Target_Name"] = ssl[0]["Target_Name"]
# 1
result["top"] = np.poly1d([y])
''' Nomenclature here is confusing:
----- Edge -- Top of current slit, bottom of prev slit
. o ' Data
===== Data
.;.;' Data
----- Edge -- Bottom of current slit, top of next slit
'''
topfun = np.poly1d([y])
xposs_top_this = np.arange(10, 2000, 100)
yposs_top_this = topfun(xposs_top_this)
initial_edges = np.array([2034], dtype=np.int)
edge = 2034
# build an array of values containing the lower edge of the slits
for target in range(len(ssl)):
# target is the slit number
edge -= DY * numslits[target]
initial_edges = np.append(initial_edges, int(edge))
# collapse the 2d flat along the walenegth axis to build a spatial profile of the slits
vertical_profile = np.mean(data, axis=1)
# build an array containing the spatial positions of the slit centers, basically the mid point between the expected
# top and bottom values of the slit pixels
spatial_centers = np.array([], dtype=np.int)
for k in np.arange(0, len(initial_edges) - 1):
spatial_centers = np.append(
spatial_centers, (initial_edges[k] + initial_edges[k + 1]) // 2)
#slit_values=np.array([])
#for k in np.arange(0, len(spatial_centers)):
# slit_values = np.append(slit_values,np.mean(vertical_profile[spatial_centers[k]-3:spatial_centers[k]+3]))
for target in range(len(ssl)):
y -= DY * numslits[target]
y = max(y, 1)
# select a 6 pixel wide section of the vertical profile around the slit center
threshold_area = vertical_profile[spatial_centers[target] -
3:spatial_centers[target] + 3]
# uses 80% of the ADU counts in the threshold area to estimate the threshold to use in defining the slits
edgeThreshold = np.mean(threshold_area) * 0.8
#if edgeThreshold > 450:
# edgeThreshold = 450
info("[%2.2i] Finding Slit Edges for %s ending at %4.0i. Slit "
"composed of %i CSU slits" %
(target, ssl[target]["Target_Name"], y, numslits[target]))
info("[%2.2i] Threshold used is %.1f" % (target, edgeThreshold))
''' First deal with the current slit '''
hpps = Wavelength.estimate_half_power_points(
bs.scislit_to_csuslit(target + 1)[0], header, bs)
if y == 1:
xposs_bot = [1024]
xposs_bot_missing = []
yposs_bot = [4.25]
botfun = np.poly1d(yposs_bot)
ok = np.where((xposs_bot > hpps[0]) & (xposs_bot < hpps[1]))
else:
(xposs_top_next, xposs_top_next_missing, yposs_top_next, xposs_bot,
xposs_bot_missing, yposs_bot,
scatter_bot_this) = find_edge_pair(data,
y,
options["edge-fit-width"],
edgeThreshold=edgeThreshold)
ok = np.where((xposs_bot > hpps[0]) & (xposs_bot < hpps[1]))
ok2 = np.where((xposs_bot_missing > hpps[0])
& (xposs_bot_missing < hpps[1]))
xposs_bot = xposs_bot[ok]
xposs_bot_missing = xposs_bot_missing[ok2]
yposs_bot = yposs_bot[ok]
if len(xposs_bot) == 0:
botfun = np.poly1d(y - DY)
else:
(botfun, bot_res, botsd,
botok) = fit_edge_poly(xposs_bot, xposs_bot_missing,
yposs_bot, options["edge-order"])
bot = botfun.c.copy()
top = topfun.c.copy()
#4
result = {}
result["Target_Name"] = ssl[target]["Target_Name"]
result["xposs_top"] = xposs_top_this
result["yposs_top"] = yposs_top_this
result["xposs_bot"] = xposs_bot
result["yposs_bot"] = yposs_bot
result["top"] = np.poly1d(top)
result["bottom"] = np.poly1d(bot)
result["hpps"] = hpps
result["ok"] = ok
results.append(result)
#5
if y == 1:
break
next = target + 2
if next > len(ssl): next = len(ssl)
hpps_next = Wavelength.estimate_half_power_points(
bs.scislit_to_csuslit(next)[0], header, bs)
ok = np.where((xposs_top_next > hpps_next[0])
& (xposs_top_next < hpps_next[1]))
ok2 = np.where((xposs_top_next_missing > hpps_next[0])
& (xposs_top_next_missing < hpps_next[1]))
xposs_top_next = xposs_top_next[ok]
xposs_top_next_missing = xposs_top_next_missing[ok2]
yposs_top_next = yposs_top_next[ok]
if len(xposs_top_next) == 0:
topfun = np.poly1d(y)
else:
(topfun, topres, topsd,
ok) = fit_edge_poly(xposs_top_next, xposs_top_next_missing,
yposs_top_next, options["edge-order"])
xposs_top_this = xposs_top_next
xposs_top_this_missing = xposs_top_next_missing
yposs_top_this = yposs_top_next
results.append({"version": options["version"]})
return results
def optimal_extraction(image, variance_image, aperture_table,
fitsfileout=None,
plotfileout=None,
plot=None):
'''Given a 2D spectrum image, a 2D variance image, and a table of apertures
(e.g. as output by find_apertures() above), this function will optimally
extract a 1D spectrum for each entry in the table of apertures.
'''
if type(image) == fits.HDUList:
hdu = image[0]
elif type(image) == fits.PrimaryHDU:
hdu = image
else:
error('Input to standard_extraction should be an HDUList or an HDU')
raise TypeError
if type(variance_image) == fits.HDUList:
vhdu = variance_image[0]
elif type(image) == fits.PrimaryHDU:
vhdu = variance_image
else:
error('Input to standard_extraction should be an HDUList or an HDU')
raise TypeError
spectra2D = hdu.data
variance2D = vhdu.data
header = hdu.header
worig = wcs.WCS(hdu.header)
## State assumptions
assert header['DISPAXIS'] == 1
assert worig.to_header()['CTYPE1'] == 'AWAV'
assert header['CD1_2'] == 0
assert header['CD2_1'] == 0
assert worig.dropaxis(1).to_header()['CTYPE1'] == 'AWAV'
## Replace old WCS in header with the collapsed WCS
w = worig.dropaxis(1)
for key in list(worig.to_header().keys()):
if key in list(header.keys()):
header.remove(key)
for key in list(w.to_header().keys()):
if key in ['PC1_1', 'CRVAL1']:
header.set(key, w.to_header()[key]*1e10,
w.to_header().comments[key])
elif key in ['CUNIT1', 'CNAME1']:
header.set(key, 'Angstrom', w.to_header().comments[key])
else:
header.set(key, w.to_header()[key],
w.to_header().comments[key])
spectra = []
variances = []
for i,row in enumerate(aperture_table):
pos = row['position']
width = int(row['width'])
info('Extracting aperture {:d} at position {:.0f}'.format(i, pos))
ymin = max([int(np.floor(pos-width)), 0])
ymax = min([int(np.ceil(pos+width)), spectra2D.shape[0]])
DmS = np.ma.MaskedArray(data=spectra2D[ymin:ymax,:],\
mask=np.isnan(spectra2D[ymin:ymax,:]))
V = np.ma.MaskedArray(data=variance2D[ymin:ymax,:],\
mask=np.isnan(spectra2D[ymin:ymax,:]))
info(' Performing standard extraction')
f_std, V_std = standard_extraction(DmS, V)
info(' Forming initial spatial profile')
P_init_data = np.array([row/f_std for row in DmS])
P_init = np.ma.MaskedArray(data=P_init_data,\
mask=np.isnan(P_init_data))
info(' Fitting spatial profile')
P = iterate_spatial_profile(P_init, DmS, V, f_std, verbose=False)
info(' Calculating optimally extracted spectrum')
f_new_denom = np.ma.MaskedArray(data=np.sum(P**2/V, axis=0),\
mask=(np.sum(P**2/V, axis=0)==0))
f_opt = np.sum(P*DmS/V, axis=0)/f_new_denom
var_fopt = np.sum(P, axis=0)/f_new_denom
sig_fopt = np.sqrt(var_fopt)
typical_sigma = np.mean(sig_fopt[~np.isnan(sig_fopt)])
spectra.append(f_opt)
variances.append(var_fopt)
info(' Typical level = {:.1f}'.format(np.mean(f_opt)))
info(' Typical sigma = {:.1f}'.format(typical_sigma))
mask = np.isnan(np.array(spectra)) | np.isnan(np.array(variances))
spectra = np.ma.MaskedArray(data=np.array(spectra), mask=mask)
variances = np.ma.MaskedArray(data=np.array(variances), mask=mask)
for i,row in enumerate(aperture_table):
hdulist = fits.HDUList([])
if plotfileout:
fig = pl.figure(figsize=(16,6))
wunit = getattr(u, w.to_header()['CUNIT1'])
sp = spectra[i]
hdulist.append(fits.PrimaryHDU(data=sp.filled(0), header=header))
hdulist[0].header['APPOS'] = row['position']
if plotfileout:
sigma = np.sqrt(variances[i])
pix = np.arange(0,sp.shape[0],1)
wavelengths = w.wcs_pix2world(pix,1)[0]*wunit.to(u.micron)*u.micron
fillmin = sp-sigma
fillmax = sp+sigma
mask = np.isnan(fillmin) | np.isnan(fillmax) | np.isnan(sp)
pl.fill_between(wavelengths[~mask], fillmin[~mask], fillmax[~mask],\
label='uncertainty',\
facecolor='black', alpha=0.2,\
linewidth=0,\
interpolate=True)
pl.plot(wavelengths, sp, 'k-',
label='Spectrum for Aperture {} at {:.0f}'.format(i,
row['position']))
pl.xlabel('Wavelength (microns)')
pl.ylabel('Flux (e-/sec)')
pl.xlim(wavelengths.value.min(),wavelengths.value.max())
pl.ylim(0,1.05*sp.max())
pl.legend(loc='best')
bn, ext = os.path.splitext(plotfileout)
plotfilename = '{}_{:02d}{}'.format(bn, i, ext)
pl.savefig(plotfilename, bbox_inches='tight')
pl.close(fig)
var = variances[i]
hdulist.append(fits.ImageHDU(data=var.filled(0), header=header))
hdulist[1].header['APPOS'] = row['position']
hdulist[1].header['COMMENT'] = 'VARIANCE DATA'
if fitsfileout:
bn, ext = os.path.splitext(fitsfileout)
fitsfilename = '{}_{:02d}{}'.format(bn, i, ext)
hdulist.writeto(fitsfilename, overwrite=True)
def handle_rectification(maskname, in_files, wavename, band_pass, files, options,
commissioning_shift=3.0, target='default', plan=None):
'''Handle slit rectification and coaddition.
Args:
maskname: The mask name string
in_files: List of stacked spectra in electron per second. Will look
like ['electrons_Offset_1.5.txt.fits', 'electrons_Offset_-1.5.txt.fits']
wavename: path (relative or full) to the wavelength stack file, string
band_pass: Band pass name, string
barset_file: Path to a mosfire fits file containing the full set of
FITS extensions for the barset. It can be any file in the list
of science files.
Returns:
None
Writes files:
[maskname]_[band]_[object name]_eps.fits --
The rectified, background subtracted, stacked eps spectrum
[maskname]_[band]_[object name]_sig.fits --
Rectified, background subtracted, stacked weight spectrum (STD/itime)
[maskname]_[band]_[object_name]_itime.fits
Rectified, CRR stacked integration time spectrum
[maskname]_[band]_[object_name]_snrs.fits
Rectified signal to noise spectrum
'''
global edges, dats, vars, itimes, shifts, lambdas, band, fidl, all_shifts
band = band_pass
dlambda = Wavelength.grating_results(band)
hpp = Filters.hpp[band]
fidl = np.arange(hpp[0], hpp[1], dlambda)
lambdas = IO.readfits(wavename, options)
if np.any(lambdas[1].data < 0) or np.any(lambdas[1].data > 29000):
info("***********WARNING ***********")
info("The file {0} may not be a wavelength file.".format(wavename))
info("Check before proceeding.")
info("***********WARNING ***********")
edges, meta = IO.load_edges(maskname, band, options)
shifts = []
posnames = []
postoshift = {}
for file in in_files:
info(":: "+str(file))
II = IO.read_drpfits(maskname, file, options)
off = np.array((II[0]["decoff"], II[0]["raoff"]),dtype=np.float64)
if "yoffset" in II[0]:
off = -II[0]["yoffset"]
else:
# Deal with data taken during commissioning
if II[0]["frameid"] == 'A': off = 0.0
else: off = commissioning_shift
try: off0
except: off0 = off
shift = off - off0
shifts.append(shift)
posnames.append(II[0]["frameid"])
postoshift[II[0]['frameid']] = shift
info("Position {0} shift: {1:2.2f} as".format(off, shift))
# this is to deal with cases in which we want to rectify one single file
if len(set(posnames)) is 1:
plans = [['A']]
else:
if plan is None:
plans = Background.guess_plan_from_positions(set(posnames))
else:
plans = plan
all_shifts = []
for myplan in plans:
to_append = []
for pos in myplan:
to_append.append(postoshift[pos])
all_shifts.append(to_append)
# Reverse the elements in all_shifts to deal with an inversion
all_shifts.reverse()
theBPM = IO.badpixelmask()
all_solutions = []
cntr = 0
if target is 'default':
outname = maskname
else:
outname = target
for plan in plans:
if len(plan) is 1:
p0 = 'A'
p1 = 'B'
else:
p0 = plan[0].replace("'", "p")
p1 = plan[1].replace("'", "p")
suffix = "%s-%s" % (p0,p1)
info("Handling plan %s" % suffix)
fname = "bsub_{0}_{1}_{2}.fits".format(outname,band,suffix)
EPS = IO.read_drpfits(maskname, fname, options)
EPS[1] = np.ma.masked_array(EPS[1], theBPM, fill_value=0)
fname = "var_{0}_{1}_{2}.fits".format(outname, band, suffix)
VAR = IO.read_drpfits(maskname, fname, options)
VAR[1] = np.ma.masked_array(VAR[1], theBPM, fill_value=np.inf)
fname = "itime_{0}_{1}_{2}.fits".format(outname, band, suffix)
ITIME = IO.read_drpfits(maskname, fname, options)
ITIME[1] = np.ma.masked_array(ITIME[1], theBPM, fill_value=0)
dats = EPS
vars = VAR
itimes = ITIME
EPS[0]["ORIGFILE"] = fname
tock = time.time()
sols = list(range(len(edges)-1,-1,-1))
shifts = all_shifts[cntr]
cntr += 1
p = Pool()
solutions = p.map(handle_rectification_helper, sols)
p.close()
all_solutions.append(solutions)
tick = time.time()
info("-----> Mask took %i. Writing to disk." % (tick-tock))
output = np.zeros((1, len(fidl)))
snrs = np.zeros((1, len(fidl)))
sdout= np.zeros((1, len(fidl)))
itout= np.zeros((1, len(fidl)))
# the barset [bs] is used for determining object position
files = IO.list_file_to_strings(files)
info("Using "+str(files[0])+" for slit configuration.")
x, x, bs = IO.readmosfits(files[0], options)
for i_slit in range(len(solutions)):
solution = all_solutions[0][i_slit]
header = EPS[0].copy()
obj = header['OBJECT']
#Again some weirdness with Longslit target names
try:
target_name = str(bs.ssl[-(i_slit+1)]['Target_Name'], 'utf-8')
except TypeError:
target_name = bs.ssl[-(i_slit+1)]['Target_Name']
header['OBJECT'] = target_name
pixel_dist = np.float(bs.ssl[-(i_slit+1)]['Target_to_center_of_slit_distance'])/0.18
pixel_dist -= solution['offset']
ll = solution["lambda"]
header["wat0_001"] = "system=world"
header["wat1_001"] = "wtype=linear"
header["wat2_001"] = "wtype=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["ctype1"] = "AWAV"
header["cunit1"] = "Angstrom"
header["crval1"] = ll[0]
header["crval2"] = -solution["eps_img"].shape[0]//2 - pixel_dist
header["crpix1"] = 1
header["crpix2"] = 1
header["cdelt1"] = ll[1]-ll[0]
header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = ll[1]-ll[0]
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = 1
S = output.shape
img = solution["eps_img"]
std = solution["sd_img"]
tms = solution["itime_img"]
for i_solution in range(1,len(all_solutions)):
info("Combining solution %i" %i_solution)
solution = all_solutions[i_solution][i_slit]
img += solution["eps_img"]
std += solution["sd_img"]
tms += solution["itime_img"]
output = np.append(output, img, 0)
output = np.append(output, np.nan*np.zeros((3,S[1])), 0)
snrs = np.append(snrs, img*tms/std, 0)
snrs = np.append(snrs, np.nan*np.zeros((3,S[1])), 0)
sdout = np.append(sdout, std, 0)
sdout = np.append(sdout, np.nan*np.zeros((3,S[1])), 0)
itout = np.append(itout, tms, 0)
itout = np.append(itout, np.nan*np.zeros((3,S[1])), 0)
header['bunit'] = ('electron/second', 'electron power')
IO.writefits(img, maskname,
"{0}_{1}_{2}_eps.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('electron/second', 'sigma/itime')
IO.writefits(std/tms, maskname,
"{0}_{1}_{2}_sig.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('second', 'exposure time')
IO.writefits(tms, maskname,
"{0}_{1}_{2}_itime.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header['bunit'] = ('', 'SNR')
IO.writefits(img*tms/std, maskname,
"{0}_{1}_{2}_snrs.fits".format(outname, band, target_name), options,
overwrite=True, header=header, lossy_compress=False)
header = EPS[0].copy()
header["wat0_001"] = "system=world"
header["wat1_001"] = "wtype=linear"
header["wat2_001"] = "wtype=linear"
header["dispaxis"] = 1
header["dclog1"] = "Transform"
header["dc-flag"] = 0
header["ctype1"] = "AWAV"
header["cunit1"] = ("Angstrom", 'Start wavelength')
header["crval1"] = ll[0]
header["crval2"] = 1
header["crpix1"] = 1
header["crpix2"] = 1
header["cdelt1"] = (ll[1]-ll[0], 'Angstrom/pixel')
header["cdelt2"] = 1
header["cname1"] = "angstrom"
header["cname2"] = "pixel"
header["cd1_1"] = (ll[1]-ll[0], 'Angstrom/pixel')
header["cd1_2"] = 0
header["cd2_1"] = 0
header["cd2_2"] = 1
header["bunit"] = "ELECTRONS/SECOND"
info("############ Final reduced file: {0}_{1}_eps.fits".format(outname,band))
IO.writefits(output, maskname, "{0}_{1}_eps.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = ""
IO.writefits(snrs, maskname, "{0}_{1}_snrs.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = "ELECTRONS/SECOND"
IO.writefits(sdout/itout, maskname, "{0}_{1}_sig.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
header["bunit"] = "SECOND"
IO.writefits(itout, maskname, "{0}_{1}_itime.fits".format(outname,
band), options, overwrite=True, header=header,
lossy_compress=False)
def imcombine(filelist, out, options, method="average", reject="none",\
lsigma=3, hsigma=3, mclip=False,\
nlow=None, nhigh=None):
'''Combines images in input list with optional rejection algorithms.
Args:
filelist: The list of files to imcombine
out: The full path to the output file
method: either "average" or "median" combine
options: Options dictionary
bpmask: The full path to the bad pixel mask
reject: none, minmax, sigclip
nlow,nhigh: Parameters for minmax rejection, see iraf docs
mclip: use median as the function to calculate the baseline values for
sigclip rejection?
lsigma, hsigma: low and high sigma rejection thresholds.
Returns:
None
Side effects:
Creates the imcombined file at location `out'
'''
assert method in ['average', 'median']
if os.path.exists(out):
os.remove(out)
if reject == 'none':
info('Combining files using ccdproc.combine task')
info(' reject=none')
for file in filelist:
debug(' Combining: {}'.format(file))
ccdproc.combine(filelist, out, method=method,\
minmax_clip=False,\
iraf_minmax_clip=True,\
sigma_clip=False,\
unit="adu")
info(' Done.')
elif reject == 'minmax':
## The IRAF imcombine minmax rejection behavior is different than the
## ccdproc minmax rejection behavior. We are using the IRAF like
## behavior here. To support this a pull request for the ccdproc
## package has been made:
## https://github.com/astropy/ccdproc/pull/358
##
## Note that the ccdproc behavior still differs slightly from the
## nominal IRAF behavior in that the rejection does not consider whether
## any of the rejected pixels have been rejected for other reasons, so
## if nhigh=1 and that pixel was masked for some other reason, the
## new ccdproc algorithm, will not mask the next highest pixel, it will
## still just mask the highest pixel even if it is already masked.
##
## From IRAF (help imcombine):
## nlow = 1, nhigh = 1 (minmax)
## The number of low and high pixels to be rejected by the
## "minmax" algorithm. These numbers are converted to fractions
## of the total number of input images so that if no rejections
## have taken place the specified number of pixels are rejected
## while if pixels have been rejected by masking, thresholding, or
## non-overlap, then the fraction of the remaining pixels,
## truncated to an integer, is used.
##
## Check that minmax rejection is possible given the number of images
if nlow is None:
nlow = 0
if nhigh is None:
nhigh = 0
if nlow + nhigh >= len(filelist):
warning('nlow + nhigh >= number of input images. Combining without rejection')
nlow = 0
nhigh = 0
if ccdproc.version.major >= 1 and ccdproc.version.minor >= 1\
and ccdproc.version.release:
info('Combining files using ccdproc.combine task')
info(' reject=clip_extrema')
info(' nlow={}'.format(nlow))
info(' nhigh={}'.format(nhigh))
for file in filelist:
info(' {}'.format(file))
ccdproc.combine(filelist, out, method=method,\
minmax_clip=False,\
clip_extrema=True,\
nlow=nlow, nhigh=nhigh,\
sigma_clip=False,\
unit="adu")
info(' Done.')
else:
## If ccdproc does not have new rejection algorithm in:
## https://github.com/astropy/ccdproc/pull/358
## Manually perform rejection using ccdproc.combiner.Combiner object
info('Combining files using local clip_extrema rejection algorithm')
info('and the ccdproc.combiner.Combiner object.')
info(' reject=clip_extrema')
info(' nlow={}'.format(nlow))
info(' nhigh={}'.format(nhigh))
for file in filelist:
info(' {}'.format(file))
ccdlist = []
for file in filelist:
ccdlist.append(ccdproc.CCDData.read(file, unit='adu', hdu=0))
c = ccdproc.combiner.Combiner(ccdlist)
nimages, nx, ny = c.data_arr.mask.shape
argsorted = np.argsort(c.data_arr.data, axis=0)
mg = np.mgrid[0:nx,0:ny]
for i in range(-1*nhigh, nlow):
where = (argsorted[i,:,:].ravel(),
mg[0].ravel(),
mg[1].ravel())
c.data_arr.mask[where] = True
if method == 'average':
result = c.average_combine()
elif method == 'median':
result = c.median_combine()
for key in list(ccdlist[0].header.keys()):
header_entry = ccdlist[0].header[key]
if key != 'COMMENT':
result.header[key] = (header_entry,
ccdlist[0].header.comments[key])
hdul = result.to_hdu()
# print(hdul)
# for hdu in hdul:
# print(type(hdu.data))
hdul[0].writeto(out)
# result.write(out)
info(' Done.')
elif reject == 'sigclip':
info('Combining files using ccdproc.combine task')
info(' reject=sigclip')
info(' mclip={}'.format(mclip))
info(' lsigma={}'.format(lsigma))
info(' hsigma={}'.format(hsigma))
baseline_func = {False: np.mean, True: np.median}
ccdproc.combine(filelist, out, method=method,\
minmax_clip=False,\
clip_extrema=False,\
sigma_clip=True,\
sigma_clip_low_thresh=lsigma,\
sigma_clip_high_thresh=hsigma,\
sigma_clip_func=baseline_func[mclip],\
sigma_clip_dev_func=np.std,\
)
info(' Done.')
else:
raise NotImplementedError('{} rejection unrecognized by MOSFIRE DRP'.format(reject))
def handle_background(filelist,
wavename,
maskname,
band_name,
options,
shifts=None,
plan=None,
extension=None,
target='default'):
'''
Perform difference imaging and subtract residual background.
The plan looks something like: [['A', 'B']]
In this case, the number of output files is equal to the length of the list (1).
If you choose to use an ABA'B' pattern then the plan will be: [["A", "B"], ["A'", "B'"]]
the background subtraction code will make and handle two files, "A-B" and "A'-B'".
'''
global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band
band = band_name
flatname = "pixelflat_2d_%s.fits" % band_name
hdr, flat = IO.readfits("pixelflat_2d_%s.fits" % (band_name), options)
if np.abs(np.median(flat) - 1) > 0.1:
error("Flat seems poorly behaved.")
raise Exception("Flat seems poorly behaved.")
'''
This next section of the code figures out the observing plan
and then deals with the bookeeping of sending the plan
to the background subtracter.
'''
hdrs = []
epss = {}
vars = {}
bss = []
times = {}
Nframes = []
i = 0
header = pf.Header()
for i in range(len(filelist)):
fl = filelist[i]
files = IO.list_file_to_strings(fl)
info("Combining observation files listed in {}".format(fl))
if shifts is None: shift = None
else: shift = shifts[i]
hdr, electron, var, bs, time, Nframe = imcombine(files,
maskname,
options,
flat,
outname="%s.fits" %
(fl),
shifts=shift,
extension=extension)
hdrs.append(hdr)
header = merge_headers(header, hdr)
epss[hdr['FRAMEID']] = electron / time
vars[hdr['FRAMEID']] = var
times[hdr['FRAMEID']] = time
bss.append(bs)
Nframes.append(Nframe)
positions = {}
i = 0
for h in hdrs:
positions[h['FRAMEID']] = i
i += 1
posnames = set(positions.keys())
if plan is None:
plan = guess_plan_from_positions(posnames)
num_outputs = len(plan)
edges, meta = IO.load_edges(maskname, band, options)
lam = IO.readfits(wavename, options)
bs = bss[0]
for i in range(num_outputs):
posname0 = plan[i][0]
posname1 = plan[i][1]
info("Handling %s - %s" % (posname0, posname1))
data = epss[posname0] - epss[posname1]
Var = vars[posname0] + vars[posname1]
itime = np.mean([times[posname0], times[posname1]], axis=0)
p = Pool()
solutions = p.map(background_subtract_helper, list(range(len(bs.ssl))))
p.close()
write_outputs(solutions,
itime,
header,
maskname,
band,
plan[i],
options,
target=target)
def imcombine(files, maskname, options, flat, outname=None, shifts=None,
extension=None):
'''
From a list of files it imcombine returns the imcombine of several values.
The imcombine code also estimates the readnoise ad RN/sqrt(numreads) so
that the variance per frame is equal to (ADU + RN^2) where RN is computed
in ADUs.
Arguments:
files[]: list of full path to files to combine
maskname: Name of mask
options: Options dictionary
flat[2048x2048]: Flat field (values should all be ~ 1.0)
outname: If set, will write (see notes below for details)
eps_[outname].fits: electron/sec file
itimes_[outname].fits: integration time
var_[outname].fits: Variance files
shifts[len(files)]: If set, will "roll" each file by the
amount in the shifts vector in pixels. This argument
is used when telescope tracking is poor. If you need
to use this, please notify Keck staff about poor
telescope tracking.
Returns 6-element tuple:
header: The combined header
electrons [2048x2048]: e- (in e- units)
var [2048x2048]: electrons + RN**2 (in e-^2 units)
bs: The MOSFIRE.Barset instance
itimes [2048x2048]: itimes (in s units)
Nframe: The number of frames that contribute to the summed
arrays above. If Nframe > 5 I use the sigma-clipping
Cosmic Ray Rejection tool. If Nframe < 5 then I drop
the max/min elements.
Notes:
header -- fits header
ADUs -- The mean # of ADUs per frame
var -- the Variance [in adu] per frame.
bs -- Barset
itimes -- The _total_ integration time in second
Nframe -- The number of frames in a stack.
Thus the number of electron per second is derived as:
e-/sec = (ADUs * Gain / Flat) * (Nframe/itimes)
The total number of electrons is:
el = ADUs * Gain * Nframe
'''
ADUs = np.zeros((len(files), 2048, 2048))
itimes = np.zeros((len(files), 2048, 2048))
prevssl = None
prevmn = None
patternid = None
maskname = None
header = None
if shifts is None:
shifts = np.zeros(len(files))
warnings.filterwarnings('ignore')
for i in range(len(files)):
fname = files[i]
thishdr, data, bs = IO.readmosfits(fname, options, extension=extension)
itimes[i,:,:] = thishdr["truitime"]
base = os.path.basename(fname).rstrip(".fits")
fnum = int(base.split("_")[1])
if shifts[i] == 0:
ADUs[i,:,:] = data.filled(0.0) / flat
else:
ADUs[i,:,:] = np.roll(data.filled(0.0) / flat, np.int(shifts[i]), axis=0)
''' Construct Header'''
if header is None:
header = thishdr
header["imfno%3.3i" % (fnum)] = (fname, "img%3.3i file name" % fnum)
list(map(lambda x: rem_header_key(header, x), ["CTYPE1", "CTYPE2", "WCSDIM",
"CD1_1", "CD1_2", "CD2_1", "CD2_2", "LTM1_1", "LTM2_2", "WAT0_001",
"WAT1_001", "WAT2_001", "CRVAL1", "CRVAL2", "CRPIX1", "CRPIX2",
"RADECSYS"]))
for card in header.cards:
if card == '': continue
key,val,comment = card
if key in thishdr:
if val != thishdr[key]:
newkey = key + ("_img%2.2i" % fnum)
try: header[newkey.rstrip()] = (thishdr[key], comment)
except: pass
''' Now handle error checking'''
if maskname is not None:
if thishdr["maskname"] != maskname:
error("File %s uses mask '%s' but the stack is of '%s'" %
(fname, thishdr["maskname"], maskname))
raise Exception("File %s uses mask '%s' but the stack is of '%s'" %
(fname, thishdr["maskname"], maskname))
maskname = thishdr["maskname"]
if thishdr["aborted"]:
error("Img '%s' was aborted and should not be used" %
fname)
raise Exception("Img '%s' was aborted and should not be used" %
fname)
if prevssl is not None:
if len(prevssl) != len(bs.ssl):
# todo Improve these checks
error("The stack of input files seems to be of "
"different masks")
raise Exception("The stack of input files seems to be of "
"different masks")
prevssl = bs.ssl
if patternid is not None:
if patternid != thishdr["frameid"]:
error("The stack should be of '%s' frames only, but "
"the current image is a '%s' frame." % (patternid,
thishdr["frameid"]))
raise Exception("The stack should be of '%s' frames only, but "
"the current image is a '%s' frame." % (patternid,
thishdr["frameid"]))
patternid = thishdr["frameid"]
if maskname is not None:
if maskname != thishdr["maskname"]:
error("The stack should be of CSU mask '%s' frames "
"only but contains a frame of '%s'." % (maskname,
thishdr["maskname"]))
raise Exception("The stack should be of CSU mask '%s' frames "
"only but contains a frame of '%s'." % (maskname,
thishdr["maskname"]))
maskname = thishdr["maskname"]
if thishdr["BUNIT"] != "ADU per coadd":
error("The units of '%s' are not in ADU per coadd and "
"this violates an assumption of the DRP. Some new code "
"is needed in the DRP to handle the new units of "
"'%s'." % (fname, thishdr["BUNIT"]))
raise Exception("The units of '%s' are not in ADU per coadd and "
"this violates an assumption of the DRP. Some new code "
"is needed in the DRP to handle the new units of "
"'%s'." % (fname, thishdr["BUNIT"]))
''' Error checking is complete'''
debug("%s %s[%s]/%s: %5.1f s, Shift: %i px" % (fname, maskname, patternid,
header['filter'], np.mean(itimes[i]), shifts[i]))
warnings.filterwarnings('always')
# the electrons and el_per_sec arrays are:
# [2048, 2048, len(files)] and contain values for
# each individual frame that is being combined.
# These need to be kept here for CRR reasons.
electrons = np.array(ADUs) * Detector.gain
el_per_sec = electrons / itimes
output = np.zeros((2048, 2048))
exptime = np.zeros((2048, 2048))
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
# Cosmic ray rejection code begins here. This code construction the
# electrons and itimes arrays.
standard = True
new_from_chuck = False
# Chuck Steidel has provided a modified version of the CRR procedure.
# to enable it, modify the variables above.
if new_from_chuck and not standard:
if len(files) >= 5:
print("Sigclip CRR")
srt = np.argsort(electrons, axis=0, kind='quicksort')
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
el_per_sec = el_per_sec[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
# Construct the mean and standard deviation by dropping the top and bottom two
# electron fluxes. This is temporary.
mean = np.mean(el_per_sec[1:-1,:,:], axis = 0)
std = np.std(el_per_sec[1:-1,:,:], axis = 0)
drop = np.where( (el_per_sec > (mean+std*4)) | (el_per_sec < (mean-std*4)) )
print("dropping: ", len(drop[0]))
electrons[drop] = 0.0
itimes[drop] = 0.0
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
else:
warning( "With less than 5 frames, the pipeline does NOT perform")
warning( "Cosmic Ray Rejection.")
# the "if false" line disables cosmic ray rejection"
if False:
for i in range(len(files)):
el = electrons[i,:,:]
it = itimes[i,:,:]
el_mf = scipy.signal.medfilt(el, 5)
bad = np.abs(el - el_mf) / np.abs(el) > 10.0
el[bad] = 0.0
it[bad] = 0.0
electrons[i,:,:] = el
itimes[i,:,:] = it
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
if standard and not new_from_chuck:
if len(files) >= 9:
info("Sigclip CRR")
srt = np.argsort(electrons, axis=0, kind='quicksort')
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
el_per_sec = el_per_sec[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
# Construct the mean and standard deviation by dropping the top and bottom two
# electron fluxes. This is temporary.
mean = np.mean(el_per_sec[2:-2,:,:], axis = 0)
std = np.std(el_per_sec[2:-2,:,:], axis = 0)
drop = np.where( (el_per_sec > (mean+std*4)) | (el_per_sec < (mean-std*4)) )
info("dropping: "+str(len(drop[0])))
electrons[drop] = 0.0
itimes[drop] = 0.0
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
elif len(files) > 5:
warning( "WARNING: Drop min/max CRR")
srt = np.argsort(el_per_sec,axis=0)
shp = el_per_sec.shape
sti = np.ogrid[0:shp[0], 0:shp[1], 0:shp[2]]
electrons = electrons[srt, sti[1], sti[2]]
itimes = itimes[srt, sti[1], sti[2]]
electrons = np.sum(electrons[1:-1,:,:], axis=0)
itimes = np.sum(itimes[1:-1,:,:], axis=0)
Nframe = len(files) - 2
else:
warning( "With less than 5 frames, the pipeline does NOT perform")
warning( "Cosmic Ray Rejection.")
# the "if false" line disables cosmic ray rejection"
if False:
for i in range(len(files)):
el = electrons[i,:,:]
it = itimes[i,:,:]
# calculate the median image
el_mf = scipy.signal.medfilt(el, 5)
el_mf_large = scipy.signal.medfilt(el_mf, 15)
# LR: this is a modified version I was experimenting with. For the version
# written by Nick, see the new_from_chuck part of this code
# sky sub
el_sky_sub = el_mf - el_mf_large
# add a constant value
el_plus_constant = el_sky_sub + 100
bad = np.abs(el - el_mf) / np.abs(el_plus_constant) > 50.0
el[bad] = 0.0
it[bad] = 0.0
electrons[i,:,:] = el
itimes[i,:,:] = it
electrons = np.sum(electrons, axis=0)
itimes = np.sum(itimes, axis=0)
Nframe = len(files)
''' Now handle variance '''
numreads = header["READS0"]
RN_adu = Detector.RN / np.sqrt(numreads) / Detector.gain
RN = Detector.RN / np.sqrt(numreads)
var = (electrons + RN**2)
''' Now mask out bad pixels '''
electrons[data.mask] = np.nan
var[data.mask] = np.inf
if "RN" in header:
error("RN Already populated in header")
raise Exception("RN Already populated in header")
header['RN'] = ("%1.3f" , "Read noise in e-")
header['NUMFRM'] = (Nframe, 'Typical number of frames in stack')
header['BUNIT'] = 'ELECTRONS/SECOND'
IO.writefits(np.float32(electrons/itimes), maskname, "eps_%s" % (outname),
options, header=header, overwrite=True)
# Update itimes after division in order to not introduce nans
itimes[data.mask] = 0.0
header['BUNIT'] = 'ELECTRONS^2'
IO.writefits(var, maskname, "var_%s" % (outname),
options, header=header, overwrite=True, lossy_compress=True)
header['BUNIT'] = 'SECOND'
IO.writefits(np.float32(itimes), maskname, "itimes_%s" % (outname),
options, header=header, overwrite=True, lossy_compress=True)
return header, electrons, var, bs, itimes, Nframe
