def apply_flat(scifilename, maskname, band): ''' Divides the contents of scifilename by the flat field and overwrites scifilename with the same file divided by the flat Args: scifilename: Path to science file name. maskname: The mask name band: The filter bands Results: Overwrites scifilename where the data contents of the file are divided by the pixel flat ''' flat = IO.load_flat(maskname, band, {}) flat_data = flat[1].filled(1.0) header, data = IO.readfits(scifilename) print("Applying flat to file {0}".format(scifilename)) IO.writefits(data / flat_data, maskname, scifilename, {}, header=header, overwrite=True)
def imcombine(filelist, maskname, fname, options, sum_type): """ combine the images in file list into fname. Sum type: rate -- filelist is in cnt/s ivar-rate -- filelist is in s/cnt snr-rate -- filelist is in SNR """ ARR = None hdr = None i = 1 itime = 0 for file in filelist: this_hdr, img = IO.readfits(file) cards = this_hdr.ascardlist() thisitime = this_hdr["truitime"] itime += thisitime if ARR is None: ARR = np.zeros(img.shape) if sum_type == "rate": ARR += img * thisitime if sum_type == "ivar-rate": ARR += thisitime / img if sum_type == "snr-rate": ARR += img * thisitime if hdr is None: hdr = this_hdr hdr.update("fno%2.2i" % i, file, "--") for card in cards: key, value, comment = (card.key, card.value, card.comment) if hdr.has_key(key) and hdr[key] != value: key = key + ("_img%2.2i" % i) if len(key) > 8: key = "HIERARCH " + key try: hdr.update(key, value, comment) except ValueError: pass hdr.update("itime", itime, "Itime for %i rectified images" % len(filelist)) if sum_type == "rate": ARR /= itime if sum_type == "ivar-rate": ARR = itime / ARR if sum_type == "snr-rate": ARR /= itime IO.writefits(ARR, maskname, fname, options, header=hdr, overwrite=True, lossy_compress=True)
def imcombine(filelist, maskname, fname, options, sum_type): ''' combine the images in file list into fname. Sum type: rate -- filelist is in cnt/s ivar-rate -- filelist is in s/cnt snr-rate -- filelist is in SNR ''' ARR = None hdr = None i = 1 itime = 0 for file in filelist: this_hdr, img = IO.readfits(file) cards = this_hdr.ascardlist() thisitime = this_hdr['truitime'] itime += thisitime if ARR is None: ARR = np.zeros(img.shape) if sum_type == 'rate': ARR += img * thisitime if sum_type == 'ivar-rate': ARR += thisitime / img if sum_type == 'snr-rate': ARR += img * thisitime if hdr is None: hdr = this_hdr hdr["fno%2.2i" % i] = (file, "--") for card in cards: key, value, comment = (card.key, card.value, card.comment) if hdr.has_key(key) and hdr[key] != value: key = key + ("_img%2.2i" % i) if len(key) > 8: key = 'HIERARCH ' + key try: hdr[key] = (value, comment) except ValueError: pass hdr['itime'] = (itime, 'Itime for %i rectified images' % len(filelist)) if sum_type == 'rate': ARR /= itime if sum_type == 'ivar-rate': ARR = itime / ARR if sum_type == 'snr-rate': ARR /= itime IO.writefits(ARR, maskname, fname, options, header=hdr, overwrite=True, lossy_compress=True)
def rectify(dname, lamdat, A, B, maskname, band, wavoptions, longoptions): header, data = IO.readfits(dname) raw_img = data * Detector.gain / header['TRUITIME'] dlam = Wavelength.grating_results(band) hpp = np.array(Filters.hpp[band]) ll_fid = np.arange(hpp[0], hpp[1], dlam) rectified = np.zeros((2048, len(ll_fid))) from scipy.interpolate import interp1d for i in xrange(2048): ll = lamdat[i, :] ss = raw_img[i, :] ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll > hpp[0]) if len(np.where(ok)[0]) < 30: continue f = interp1d(ll[ok], ss[ok], bounds_error=False) rectified[i, :] = f(ll_fid) header.update("wat0_001", "system=world") header.update("wat1_001", "wtype=linear") header.update("wat2_001", "wtype=linear") header.update("dispaxis", 1) header.update("dclog1", "Transform") header.update("dc-flag", 0) header.update("ctype1", "AWAV") header.update("cunit1", "Angstrom") header.update("crval1", ll_fid[0]) header.update("crval2", 0) header.update("crpix1", 1) header.update("crpix2", 1) header.update("cdelt1", 1) header.update("cdelt2", 1) header.update("cname1", "angstrom") header.update("cname2", "pixel") header.update("cd1_1", dlam) header.update("cd1_2", 0) header.update("cd2_1", 0) header.update("cd2_2", 1) header.update("object", "rectified [eps]") IO.writefits(rectified, maskname, "rectified_%s" % (dname), wavoptions, header=header, overwrite=True, lossy_compress=True)
def audit(filename): header, data = IO.readfits(filename) ll0 = header['crval1'] dlam = header['cd1_1'] ls = ll0 + dlam * np.arange(data.shape[1]) linelist = Wavelength.pick_linelist(header) deltas = [] sigs = [] xpos = [] ys = [] for y in np.linspace(750, 1100, 30): #for y in np.linspace(5, 640, 50): sp = np.ma.array(data[y,:]) xs, sxs, sigmas = Wavelength.find_known_lines(linelist, ls, sp, Options.wavelength) xpos.append(xs) ys.append([y] * len(xs)) deltas.append(xs - (linelist - ll0)/dlam) sigs.append(sxs) xpos, ys, deltas, sigs = map(np.array, [xpos, ys, deltas, sigs]) deltas[np.abs(deltas) > .75] = np.nan sigs[np.abs(sigs) > .001] = np.nan pl.clf() size = 0.003/sigs size[size > 30] = 30 size[size < 1] = 1 pl.scatter( xpos, ys, c=deltas, s=size) pl.xlim([0, data.shape[1]]) pl.ylim([0, data.shape[0]]) pl.xlabel("Spectral pixel") pl.ylabel("Spatial pixel") pl.title("Night sky line deviation from solution [pixel]") pl.colorbar() pl.savefig("audit.pdf") pdb.set_trace()
def rectify(dname, lamdat, A, B, maskname, band, wavoptions, longoptions): header, data = IO.readfits(dname) raw_img = data * Detector.gain / header['TRUITIME'] dlam = Wavelength.grating_results(band) hpp = np.array(Filters.hpp[band]) ll_fid = np.arange(hpp[0], hpp[1], dlam) rectified = np.zeros((2048, len(ll_fid))) from scipy.interpolate import interp1d for i in xrange(2048): ll = lamdat[i,:] ss = raw_img[i,:] ok = np.isfinite(ll) & np.isfinite(ss) & (ll < hpp[1]) & (ll > hpp[0]) if len(np.where(ok)[0]) < 30: continue f = interp1d(ll[ok], ss[ok], bounds_error=False) rectified[i,:] = f(ll_fid) header.update("wat0_001", "system=world") header.update("wat1_001", "wtype=linear") header.update("wat2_001", "wtype=linear") header.update("dispaxis", 1) header.update("dclog1", "Transform") header.update("dc-flag", 0) header.update("ctype1", "AWAV") header.update("cunit1", "Angstrom") header.update("crval1", ll_fid[0]) header.update("crval2", 0) header.update("crpix1", 1) header.update("crpix2", 1) header.update("cdelt1", 1) header.update("cdelt2", 1) header.update("cname1", "angstrom") header.update("cname2", "pixel") header.update("cd1_1", dlam) header.update("cd1_2", 0) header.update("cd2_1", 0) header.update("cd2_2", 1) header.update("object", "rectified [eps]") IO.writefits(rectified, maskname, "rectified_%s" % (dname), wavoptions, header=header, overwrite=True, lossy_compress=True)
def audit(filename): header, data = IO.readfits(filename) ll0 = header['crval1'] dlam = header['cd1_1'] ls = ll0 + dlam * np.arange(data.shape[1]) linelist = Wavelength.pick_linelist(header) deltas = [] sigs = [] xpos = [] ys = [] for y in np.linspace(750, 1100, 30): #for y in np.linspace(5, 640, 50): sp = np.ma.array(data[y, :]) xs, sxs, sigmas = Wavelength.find_known_lines(linelist, ls, sp, Options.wavelength) xpos.append(xs) ys.append([y] * len(xs)) deltas.append(xs - (linelist - ll0) / dlam) sigs.append(sxs) xpos, ys, deltas, sigs = map(np.array, [xpos, ys, deltas, sigs]) deltas[np.abs(deltas) > .75] = np.nan sigs[np.abs(sigs) > .001] = np.nan pl.clf() size = 0.003 / sigs size[size > 30] = 30 size[size < 1] = 1 pl.scatter(xpos, ys, c=deltas, s=size) pl.xlim([0, data.shape[1]]) pl.ylim([0, data.shape[0]]) pl.xlabel("Spectral pixel") pl.ylabel("Spatial pixel") pl.title("Night sky line deviation from solution [pixel]") pl.colorbar() pl.savefig("audit.pdf") pdb.set_trace()
def apply_flat(scifilename, maskname, band): ''' Divides the contents of scifilename by the flat field and overwrites scifilename with the same file divided by the flat Args: scifilename: Path to science file name. maskname: The mask name band: The filter bands Results: Overwrites scifilename where the data contents of the file are divided by the pixel flat ''' flat = IO.load_flat(maskname, band, {}) flat_data = flat[1].filled(1.0) header, data = IO.readfits(scifilename) print("Applying flat to file {0}".format(scifilename)) IO.writefits(data/flat_data, maskname, scifilename, {}, header=header, overwrite=True)
def handle_rectification(maskname, in_files, wavename, band_pass, files, options, commissioning_shift=3.0, target='default'): '''Handle slit rectification and coaddition. Args: maskname: The mask name string in_files: List of stacked spectra in electron per second. Will look like ['electrons_Offset_1.5.txt.fits', 'electrons_Offset_-1.5.txt.fits'] wavename: path (relative or full) to the wavelength stack file, string band_pass: Band pass name, string barset_file: Path to a mosfire fits file containing the full set of FITS extensions for the barset. It can be any file in the list of science files. Returns: None Writes files: [maskname]_[band]_[object name]_eps.fits -- The rectified, background subtracted, stacked eps spectrum [maskname]_[band]_[object name]_sig.fits -- Rectified, background subtracted, stacked weight spectrum (STD/itime) [maskname]_[band]_[object_name]_itime.fits Rectified, CRR stacked integration time spectrum [maskname]_[band]_[object_name]_snrs.fits Rectified signal to noise spectrum ''' global edges, dats, vars, itimes, shifts, lambdas, band, fidl, all_shifts band = band_pass dlambda = Wavelength.grating_results(band) hpp = Filters.hpp[band] fidl = np.arange(hpp[0], hpp[1], dlambda) lambdas = IO.readfits(wavename, options) if np.any(lambdas[1].data < 0) or np.any(lambdas[1].data > 29000): info("***********WARNING ***********") info("The file {0} may not be a wavelength file.".format(wavename)) info("Check before proceeding.") info("***********WARNING ***********") edges, meta = IO.load_edges(maskname, band, options) shifts = [] posnames = [] postoshift = {} for file in in_files: info(":: "+str(file)) II = IO.read_drpfits(maskname, file, options) off = np.array((II[0]["decoff"], II[0]["raoff"]),dtype=np.float64) if "yoffset" in II[0]: off = -II[0]["yoffset"] else: # Deal with data taken during commissioning if II[0]["frameid"] == 'A': off = 0.0 else: off = commissioning_shift try: off0 except: off0 = off shift = off - off0 shifts.append(shift) posnames.append(II[0]["frameid"]) postoshift[II[0]['frameid']] = shift info("Position {0} shift: {1:2.2f} as".format(off, shift)) # this is to deal with cases in which we want to rectify one single file if len(set(posnames)) is 1: plans = [['A']] else: plans = Background.guess_plan_from_positions(set(posnames)) all_shifts = [] for plan in plans: to_append = [] for pos in plan: to_append.append(postoshift[pos]) all_shifts.append(to_append) # Reverse the elements in all_shifts to deal with an inversion all_shifts.reverse() theBPM = IO.badpixelmask() all_solutions = [] cntr = 0 if target is 'default': outname = maskname else: outname = target for plan in plans: if len(plan) is 1: p0 = 'A' p1 = 'B' else: p0 = plan[0].replace("'", "p") p1 = plan[1].replace("'", "p") suffix = "%s-%s" % (p0,p1) info("Handling plan %s" % suffix) fname = "bsub_{0}_{1}_{2}.fits".format(outname,band,suffix) EPS = IO.read_drpfits(maskname, fname, options) EPS[1] = np.ma.masked_array(EPS[1], theBPM, fill_value=0) fname = "var_{0}_{1}_{2}.fits".format(outname, band, suffix) VAR = IO.read_drpfits(maskname, fname, options) VAR[1] = np.ma.masked_array(VAR[1], theBPM, fill_value=np.inf) fname = "itime_{0}_{1}_{2}.fits".format(outname, band, suffix) ITIME = IO.read_drpfits(maskname, fname, options) ITIME[1] = np.ma.masked_array(ITIME[1], theBPM, fill_value=0) dats = EPS vars = VAR itimes = ITIME EPS[0]["ORIGFILE"] = fname tock = time.time() sols = range(len(edges)-1,-1,-1) shifts = all_shifts[cntr] cntr += 1 p = Pool() solutions = p.map(handle_rectification_helper, sols) p.close() all_solutions.append(solutions) tick = time.time() info("-----> Mask took %i. Writing to disk." % (tick-tock)) output = np.zeros((1, len(fidl))) snrs = np.zeros((1, len(fidl))) sdout= np.zeros((1, len(fidl))) itout= np.zeros((1, len(fidl))) # the barset [bs] is used for determining object position files = IO.list_file_to_strings(files) info("Using "+str(files[0])+" for slit configuration.") x, x, bs = IO.readmosfits(files[0], options) for i_slit in xrange(len(solutions)): solution = all_solutions[0][i_slit] header = EPS[0].copy() obj = header['OBJECT'] target_name = bs.ssl[-(i_slit+1)]['Target_Name'] header['OBJECT'] = target_name pixel_dist = np.float(bs.ssl[-(i_slit+1)]['Target_to_center_of_slit_distance'])/0.18 pixel_dist -= solution['offset'] ll = solution["lambda"] header["wat0_001"] = "system=world" header["wat1_001"] = "wtype=linear" header["wat2_001"] = "wtype=linear" header["dispaxis"] = 1 header["dclog1"] = "Transform" header["dc-flag"] = 0 header["ctype1"] = "AWAV" header["cunit1"] = "Angstrom" header["crval1"] = ll[0] header["crval2"] = -solution["eps_img"].shape[0]/2 - pixel_dist header["crpix1"] = 1 header["crpix2"] = 1 #remove redundant CDELTi due to wavelength issues with ds9 #see: https://github.com/Keck-DataReductionPipelines/MosfireDRP/issues/44 #header["cdelt1"] = 1 #header["cdelt2"] = 1 header["cname1"] = "angstrom" header["cname2"] = "pixel" header["cd1_1"] = ll[1]-ll[0] header["cd1_2"] = 0 header["cd2_1"] = 0 header["cd2_2"] = 1 try: header["BARYCORR"]= (lambdas[0]['BARYCORR'],lambdas[0].comments['BARYCORR']) except KeyError: warning( "Barycentric corrections not applied to the wavelength solution") pass S = output.shape img = solution["eps_img"] std = solution["sd_img"] tms = solution["itime_img"] for i_solution in xrange(1,len(all_solutions)): info("Combining solution %i" %i_solution) solution = all_solutions[i_solution][i_slit] img += solution["eps_img"] std += solution["sd_img"] tms += solution["itime_img"] #print "adding in quadrature" output = np.append(output, img, 0) output = np.append(output, np.nan*np.zeros((3,S[1])), 0) snrs = np.append(snrs, img*tms/std, 0) snrs = np.append(snrs, np.nan*np.zeros((3,S[1])), 0) sdout = np.append(sdout, std, 0) sdout = np.append(sdout, np.nan*np.zeros((3,S[1])), 0) itout = np.append(itout, tms, 0) itout = np.append(itout, np.nan*np.zeros((3,S[1])), 0) header['bunit'] = ('electron/second', 'electron power') IO.writefits(img, maskname, "{0}_{1}_{2}_eps.fits".format(outname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header['bunit'] = ('electron/second', 'sigma/itime') IO.writefits(std/tms, maskname, "{0}_{1}_{2}_sig.fits".format(outname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header['bunit'] = ('second', 'exposure time') IO.writefits(tms, maskname, "{0}_{1}_{2}_itime.fits".format(outname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header['bunit'] = ('', 'SNR') IO.writefits(img*tms/std, maskname, "{0}_{1}_{2}_snrs.fits".format(outname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header = EPS[0].copy() header["wat0_001"] = "system=world" header["wat1_001"] = "wtype=linear" header["wat2_001"] = "wtype=linear" header["dispaxis"] = 1 header["dclog1"] = "Transform" header["dc-flag"] = 0 header["ctype1"] = "AWAV" header["cunit1"] = ("Angstrom", 'Start wavelength') header["crval1"] = ll[0] header["crval2"] = 1 header["crpix1"] = 1 header["crpix2"] = 1 #remove redundant CDELTi due to wavelength issues with ds9 #see: https://github.com/Keck-DataReductionPipelines/MosfireDRP/issues/44 #header["cdelt1"] = 1 #header["cdelt2"] = 1 header["cname1"] = "angstrom" header["cname2"] = "pixel" header["cd1_1"] = (ll[1]-ll[0], 'Angstrom/pixel') header["cd1_2"] = 0 header["cd2_1"] = 0 header["cd2_2"] = 1 try: header["BARYCORR"]= (lambdas[0]['BARYCORR'],lambdas[0].comments['BARYCORR']) except KeyError: warning( "Barycentric corrections not applied to the wavelength solution") pass header["bunit"] = "ELECTRONS/SECOND" info("############ Final reduced file: {0}_{1}_eps.fits".format(outname,band)) IO.writefits(output, maskname, "{0}_{1}_eps.fits".format(outname, band), options, overwrite=True, header=header, lossy_compress=False) header["bunit"] = "" IO.writefits(snrs, maskname, "{0}_{1}_snrs.fits".format(outname, band), options, overwrite=True, header=header, lossy_compress=False) header["bunit"] = "ELECTRONS/SECOND" IO.writefits(sdout/itout, maskname, "{0}_{1}_sig.fits".format(outname, band), options, overwrite=True, header=header, lossy_compress=False) header["bunit"] = "SECOND" IO.writefits(itout, maskname, "{0}_{1}_itime.fits".format(outname, band), options, overwrite=True, header=header, lossy_compress=False)
def go(fname): global plot_arr print fname (header, data) = IO.readfits(fname) bs = CSU.Barset() bs.set_header(header) bars = [] means = [] sds = [] deltas = [] deltas_mm = [] poss = [] poss_mm = [] poss_y = [] request = [] qs = [] bars = range(1, 93) fit_fun = Fit.residual_single for bar in bars: pos = bs.get_bar_pix(bar) if bar % 8 == 0: print "%2.0i: (%7.2f, %7.2f)" % (bar, pos[0], pos[1]) if is_odd(bar): if (bs.pos[bar] - bs.pos[bar-1]) < 2.7: fit_fun = Fit.residual_pair else: fit_fun = Fit.residual_single width = 19 [[xslice, yslice],extent,ystart] = make_slice(pos,0,width,30) if not is_in_bounds(extent): fits = [0,0,0,0,0] [ff,ok] = [np.poly1d(0,0), []] means.append(fits) sds.append(fits) drop_this = True else: drop_this = False fits = [] ys = np.arange(-10,10, dtype=np.float32) for i in ys: tofit = data[ystart-i, xslice] y = median_tails(tofit) ps = Fit.do_fit(y, fit_fun) fits.append(ps[0]) fits = np.array(fits) fits[:,1] += 1 # fit to the ridgeline [ff, ok] = fit_line_with_sigclip(ys, fits[:,1]) m = [np.mean(fits[:,i]) for i in range(5)] s = [np.std(fits[:,i]) for i in range(5)] means.append(m) sds.append(s) slit_center_offset = pos[1] - ystart fc = ff(slit_center_offset) slit_center_pos = np.float(extent[0] + fc ) if drop_this: poss.append(NaN) poss_y.append(NaN) poss_mm.append(NaN) else: poss.append(slit_center_pos) poss_y.append(ystart) poss_mm.append(CSU.csu_pix_to_mm_poly(slit_center_pos, ystart)[0]) delta = np.float(slit_center_pos - pos[0]) if drop_this: deltas.append(NaN) deltas_mm.append(NaN) else: deltas.append(delta) b = CSU.csu_pix_to_mm_poly(slit_center_pos + delta, ystart)[0] deltas_mm.append(b - poss_mm[-1]) q = np.float(np.degrees(np.tan(ff(1)-ff(0)))) if drop_this: qs.append(NaN) qs.append(q) means = np.array(means) f = lambda x: np.array(x).ravel() sds = f(sds) deltas = f(deltas) poss = f(poss) poss_y = f(poss_y) poss_mm = f(poss_mm) deltas_mm = f(deltas_mm) qs = f(qs) bars = f(bars) fout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".sav" print "saving" tosav = {"bars": bars, "request": bs.pos, "deltas_mm": deltas_mm, "poss": poss, "poss_mm": poss_mm, "deltas": deltas, "means": means, "qs": qs} scipy.io.savemat(fout, tosav) save_region(bs, "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".reg") print "saved" regout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".meas.reg" pairs = np.array([poss,poss_y]).transpose() s = CSU.to_ds9_region(pairs, dash=0, color="blue", label=False) try: f = open(regout, "w") f.writelines(s) f.close() except: print "Couldn't write: %s" % regout return [tosav, bs]
x1 = pos[0] + w if x1 > 2047: x1 = 2047 if x0 > x1: x0 = x1 xs = slice(x0, x1) y0 = pos[1] - h if y0 < 0: y0 = 0 y1 = pos[1] + h if y1 > 2047: y1 = 2047 if y0 > y1: y0 = y1 ys = slice(y0, y1) return [[xs, ys], [x0, x1, y0, y1]] (header, data) = IO.readfits("/users/npk/desktop/c8/m101029_0233.ref.fits") (header2, data2) = IO.readfits("/users/npk/desktop/c8/m101029_0425.ref.fits") (header3, data3) = IO.readfits("/users/npk/desktop/c8/m101029_0427.ref.fits") data = data3 deg = np.pi / 180. np.set_printoptions(precision=3) np.set_printoptions(suppress=True) reload(CSU) reload(IO) reload(Fit) reload(Detector) pl.ion()
import numpy as np import time import pylab as pl import scipy as sp from scipy import interpolate as II import spline import MOSFIRE.Options as options from MOSFIRE import IO, Fit, Wavelength path = "/scr2/mosfire/c9_npk/npk_calib4_q1700_pa_0/" mdat = IO.readmosfits(path + "m110323_2737.fits") fdat = IO.readfits(path + "pixelflat_2d_H.fits") ldat = IO.readfits(path + "lambda_solution_m110323_2737.fits") dat = mdat[1]/fdat[1] lam = ldat[1] yroi=slice(707,736) slit = dat[yroi,:] lslit = lam[yroi,:] DRAW = False if DRAW: pl.figure(1) pl.clf() pl.subplot(2,2,1) lamroi = slice(500,600)
def handle_flats(flatlist, maskname, band, options, extension=None,edgeThreshold=450,lampOffList=None,longslit=None): ''' handle_flats is the primary entry point to the Flats module. handle_flats takes a list of individual exposure FITS files and creates: 1. A CRR, dark subtracted, pixel-response flat file. 2. A set of polynomials that mark the edges of a slit Inputs: flatlist: maskname: The name of a mask band: A string indicating the bandceil Outputs: file {maskname}/flat_2d_{band}.fits -- pixel response flat file {maskname}/edges.np ''' tick = time.time() # Check bpos = np.ones(92) * -1 #Retrieve the list of files to use for flat creation. flatlist = IO.list_file_to_strings(flatlist) # Print the filenames to Standard-out for flat in flatlist: info(str(flat)) #Determine if flat files headers are in agreement for fname in flatlist: hdr, dat, bs = IO.readmosfits(fname, options, extension=extension) try: bs0 except: bs0 = bs if np.any(bs0.pos != bs.pos): print "bs0: "+str(bs0.pos)+" bs: "+str(bs.pos) error("Barset do not seem to match") raise Exception("Barsets do not seem to match") if hdr["filter"] != band: error ("Filter name %s does not match header filter name " "%s in file %s" % (band, hdr["filter"], fname)) raise Exception("Filter name %s does not match header filter name " "%s in file %s" % (band, hdr["filter"], fname)) for i in xrange(len(bpos)): b = hdr["B{0:02d}POS".format(i+1)] if bpos[i] == -1: bpos[i] = b else: if bpos[i] != b: error("Bar positions are not all the same in " "this set of flat files") raise Exception("Bar positions are not all the same in " "this set of flat files") bs = bs0 # Imcombine the lamps ON flats info("Attempting to combine previous files") combine(flatlist, maskname, band, options) # Imcombine the lamps OFF flats and subtract the off from the On sets if lampOffList != None: #Retrieve the list of files to use for flat creation. lampOffList = IO.list_file_to_strings(lampOffList) # Print the filenames to Standard-out for flat in lampOffList: info(str(flat)) print "Attempting to combine Lamps off data" combine(lampOffList, maskname, band, options, lampsOff=True) combine_off_on( maskname, band, options) debug("Combined '%s' to '%s'" % (flatlist, maskname)) info("Comgined to '%s'" % (maskname)) path = "combflat_2d_%s.fits" % band (header, data) = IO.readfits(path, use_bpm=True) info("Flat written to %s" % path) # Edge Trace if bs.long_slit: info( "Long slit mode recognized") info( "Central row position: "+str(longslit["row_position"])) info( "Upper and lower limits: "+str(longslit["yrange"][0])+" "+str(longslit["yrange"][1])) results = find_longslit_edges(data,header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit) elif bs.long2pos_slit: info( "Long2pos mode recognized") results = find_long2pos_edges(data,header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit) else: results = find_and_fit_edges(data, header, bs, options,edgeThreshold=edgeThreshold) results[-1]["maskname"] = maskname results[-1]["band"] = band np.save("slit-edges_{0}".format(band), results) save_ds9_edges(results, options) # Generate Flat out = "pixelflat_2d_%s.fits" % (band) if lampOffList != None: make_pixel_flat(data, results, options, out, flatlist, lampsOff=True) else: make_pixel_flat(data, results, options, out, flatlist, lampsOff=False) info( "Pixel flat took {0:6.4} s".format(time.time()-tick))
def handle_background(filelist, wavename, maskname, band_name, options, shifts=None, plan=None, extension=None): ''' Perform difference imaging and subtract residual background. The plan looks something like: [['A', 'B']] In this case, the number of output files is equal to the length of the list (1). If you choose to use an ABA'B' pattern then the plan will be: [["A", "B"], ["A'", "B'"]] the background subtraction code will make and handle two files, "A-B" and "A'-B'". ''' global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band band = band_name flatname = "pixelflat_2d_%s.fits" % band_name hdr, flat = IO.readfits("pixelflat_2d_%s.fits" % (band_name), options) if np.abs(np.median(flat) - 1) > 0.1: raise Exception("Flat seems poorly behaved.") ''' This next section of the code figures out the observing plan and then deals with the bookeeping of sending the plan to the background subtracter. ''' hdrs = [] epss = {} vars = {} bss = [] times = {} Nframes = [] i = 0 header = pf.Header() for i in xrange(len(filelist)): fl = filelist[i] files = IO.list_file_to_strings(fl) print "Combining" if shifts is None: shift = None else: shift = shifts[i] hdr, electron, var, bs, time, Nframe = imcombine(files, maskname, options, flat, outname="%s.fits" % (fl), shifts=shift, extension=extension) hdrs.append(hdr) header = merge_headers(header, hdr) epss[hdr['FRAMEID']] = electron / time vars[hdr['FRAMEID']] = var times[hdr['FRAMEID']] = time bss.append(bs) Nframes.append(Nframe) positions = {} i = 0 for h in hdrs: positions[h['FRAMEID']] = i i += 1 posnames = set(positions.keys()) if plan is None: plan = guess_plan_from_positions(posnames) num_outputs = len(plan) edges, meta = IO.load_edges(maskname, band, options) lam = IO.readfits(wavename, options) bs = bss[0] for i in xrange(num_outputs): posname0 = plan[i][0] posname1 = plan[i][1] print "Handling %s - %s" % (posname0, posname1) data = epss[posname0] - epss[posname1] Var = vars[posname0] + vars[posname1] itime = np.mean([times[posname0], times[posname1]], axis=0) p = Pool() solutions = p.map(background_subtract_helper, xrange(len(bs.ssl))) p.close() write_outputs(solutions, itime, header, maskname, band, plan[i], options)
def go(maskname, band, filenames, wavefile, wavoptions, longoptions, use_flat=False): ''' The go command is the main entry point into this module. Inputs: maskname: String of the mask name band: String of 'Y', 'J', 'H', or 'K' filenames: List of filenames to reduce wavefile: String of path to FITS file with the wavelength solution wavoptions: The Wavelength Options dictionary longoptions: Dictionary containing: {'yrange': The pixel range to extract over 'row_position': The row to solve the initial wavelength solution on} use_flat: Boolean False [default] means to use no flat field Boolean True means to divide by the pixelflat ''' wavename = Wavelength.filelist_to_wavename(filenames, band, maskname, wavoptions).rstrip(".fits") print "Wavefile: {0}".format(wavefile) lamhdr, lamdat = IO.readfits(wavefile) positions = [] objname = None for listfile in filenames: fnames = IO.list_file_to_strings(listfile) if len(fnames) != 1: raise Exception("I currently expect only one file per position. Remove multiple entries and try again") header, data, bs = IO.readmosfits(fnames[0], wavoptions) if objname is None: objname = header["object"] if objname != header["object"]: print ("Trying to combine longslit stack of object {0} " "with object {1}".format(objname, header["object"])) print("{0:18s} {1:30s} {2:2s} {3:4.1f}".format(file, header["object"], header["frameid"], header["yoffset"])) positions.append([fnames[0], header, data, bs]) print("{0:2g} nod positions found. Producing stacked difference" \ " image.".format(len(positions))) for i in xrange(len(positions)-1): A = positions[i] B = positions[i+1] print("----------- -----".format(A,B)) dname, varname = imdiff(A, B, maskname, band, header, wavoptions) if use_flat: apply_flat(dname, maskname, band) apply_flat(varname, maskname, band) rectify(dname, lamdat, A, B, maskname, band, wavoptions, longoptions) rectify(varname, lamdat, A, B, maskname, band, wavoptions, longoptions) print dname dname, vname = imdiff(B, A, maskname, band, header, wavoptions) if use_flat: apply_flat(dname, maskname, band) apply_flat(vname, maskname, band) rectify(dname, lamdat, B, A, maskname, band, wavoptions, longoptions) rectify(vname, lamdat, B, A, maskname, band, wavoptions, longoptions) if False: fname = os.path.join(path, wavename + ".fits") B = IO.readfits(fname) B = [fname, B[0], B[1]] for i in xrange(len(positions)): A = positions[i] imdiff(A, B, maskname, band, wavoptions) rectify(path, dname, lamdat, A, B, maskname, band, wavoptions, longoptions) imdiff(B, A, maskname, band, wavoptions) rectify(path, dname, lamdat, B, A, maskname, band, wavoptions, longoptions)
if x0 < 0: x0 = 0 x1 = pos[0] + w if x1 > 2047: x1 = 2047 if x0 > x1: x0 = x1 xs = slice(x0, x1) y0 = pos[1]-h if y0 < 0: y0 = 0 y1 = pos[1]+h if y1 > 2047: y1 = 2047 if y0 > y1: y0 = y1 ys = slice(y0,y1) return [[xs,ys],[x0,x1,y0,y1]] (header, data) = IO.readfits("/users/npk/desktop/c8/m101029_0233.ref.fits") (header2, data2) = IO.readfits("/users/npk/desktop/c8/m101029_0425.ref.fits") (header3, data3) = IO.readfits("/users/npk/desktop/c8/m101029_0427.ref.fits") data = data3 deg = np.pi/180. np.set_printoptions(precision=3) np.set_printoptions(suppress=True) reload(CSU) reload(IO) reload(Fit) reload(Detector)
import scipy.ndimage import matplotlib import pyfits as pf import MOSFIRE.Options as options from MOSFIRE import IO, Fit, Wavelength pl.ion() path = "/users/npk/desktop/c9_reduce/npk_calib3_q1700_pa_0/" path = "/scr2/mosfire/c9_npk/npk_calib3_q1700_pa_0/" path = "/scr2/mosfire/firstlight/NGC5053" mdat = IO.readmosfits(path + "m120406_0291.fits") fdat = IO.readfits(path + "pixelflat_2d_J.fits") ldat = IO.readfits(path + "lambda_solution_m120406_0291.fits") gain = 2.15 dat = mdat[1]/fdat[1] * gain dat = mdat[1] * gain lam = ldat[1] dat[np.logical_not(np.isfinite(dat))] = 0. yroi=slice(86, 160) yroi=slice(173, 203) yroi=slice(1015, 1090) xroi=slice(0,2048)
def go(maskname, band, filenames, wavefile, wavoptions, longoptions, use_flat=False): ''' The go command is the main entry point into this module. Inputs: maskname: String of the mask name band: String of 'Y', 'J', 'H', or 'K' filenames: List of filenames to reduce wavefile: String of path to FITS file with the wavelength solution wavoptions: The Wavelength Options dictionary longoptions: Dictionary containing: {'yrange': The pixel range to extract over 'row_position': The row to solve the initial wavelength solution on} use_flat: Boolean False [default] means to use no flat field Boolean True means to divide by the pixelflat ''' wavename = Wavelength.filelist_to_wavename(filenames, band, maskname, wavoptions).rstrip(".fits") print "Wavefile: {0}".format(wavefile) lamhdr, lamdat = IO.readfits(wavefile) positions = [] objname = None for listfile in filenames: fnames = IO.list_file_to_strings(listfile) if len(fnames) != 1: raise Exception( "I currently expect only one file per position. Remove multiple entries and try again" ) header, data, bs = IO.readmosfits(fnames[0], wavoptions) if objname is None: objname = header["object"] if objname != header["object"]: print( "Trying to combine longslit stack of object {0} " "with object {1}".format(objname, header["object"])) print("{0:18s} {1:30s} {2:2s} {3:4.1f}".format(file, header["object"], header["frameid"], header["yoffset"])) positions.append([fnames[0], header, data, bs]) print("{0:2g} nod positions found. Producing stacked difference" \ " image.".format(len(positions))) for i in xrange(len(positions) - 1): A = positions[i] B = positions[i + 1] print("----------- -----".format(A, B)) dname, varname = imdiff(A, B, maskname, band, header, wavoptions) if use_flat: apply_flat(dname, maskname, band) apply_flat(varname, maskname, band) rectify(dname, lamdat, A, B, maskname, band, wavoptions, longoptions) rectify(varname, lamdat, A, B, maskname, band, wavoptions, longoptions) print dname dname, vname = imdiff(B, A, maskname, band, header, wavoptions) if use_flat: apply_flat(dname, maskname, band) apply_flat(vname, maskname, band) rectify(dname, lamdat, B, A, maskname, band, wavoptions, longoptions) rectify(vname, lamdat, B, A, maskname, band, wavoptions, longoptions) if False: fname = os.path.join(path, wavename + ".fits") B = IO.readfits(fname) B = [fname, B[0], B[1]] for i in xrange(len(positions)): A = positions[i] imdiff(A, B, maskname, band, wavoptions) rectify(path, dname, lamdat, A, B, maskname, band, wavoptions, longoptions) imdiff(B, A, maskname, band, wavoptions) rectify(path, dname, lamdat, B, A, maskname, band, wavoptions, longoptions)
def handle_rectification(maskname, in_files, wavename, band_pass, barset_file, options, commissioning_shift=3.0): '''Handle slit rectification and coaddition. Args: maskname: The mask name string in_files: List of stacked spectra in electron per second. Will look like ['electrons_Offset_1.5.txt.fits', 'electrons_Offset_-1.5.txt.fits'] wavename: path (relative or full) to the wavelength stack file, string band_pass: Band pass name, string barset_file: Path to a mosfire fits file containing the full set of FITS extensions for the barset. It can be any file in the list of science files. Returns: None Writes files: [maskname]_[band]_[object name]_eps.fits -- The rectified, background subtracted, stacked eps spectrum [maskname]_[band]_[object name]_sig.fits -- Rectified, background subtracted, stacked weight spectrum (STD/itime) [maskname]_[band]_[object_name]_itime.fits Rectified, CRR stacked integration time spectrum [maskname]_[band]_[object_name]_snrs.fits Rectified signal to noise spectrum ''' global edges, dats, vars, itimes, shifts, lambdas, band, fidl, all_shifts band = band_pass dlambda = Wavelength.grating_results(band) hpp = Filters.hpp[band] fidl = np.arange(hpp[0], hpp[1], dlambda) lambdas = IO.readfits(wavename, options) if np.any(lambdas[1].data < 0) or np.any(lambdas[1].data > 29000): print "***********WARNING ***********" print "The file {0} may not be a wavelength file.".format(wavename) print "Check before proceeding." print "***********WARNING ***********" edges, meta = IO.load_edges(maskname, band, options) shifts = [] posnames = [] postoshift = {} for file in in_files: print ":: ", file II = IO.read_drpfits(maskname, file, options) off = np.array((II[0]["decoff"], II[0]["raoff"]),dtype=np.float64) if "yoffset" in II[0]: off = -II[0]["yoffset"] else: # Deal with data taken during commissioning if II[0]["frameid"] == 'A': off = 0.0 else: off = commissioning_shift try: off0 except: off0 = off shift = off - off0 shifts.append(shift) posnames.append(II[0]["frameid"]) postoshift[II[0]['frameid']] = shift print "Position {0} shift: {1:2.2f} as".format(off, shift) plans = Background.guess_plan_from_positions(set(posnames)) all_shifts = [] for plan in plans: to_append = [] for pos in plan: to_append.append(postoshift[pos]) all_shifts.append(to_append) # Reverse the elements in all_shifts to deal with an inversion all_shifts.reverse() theBPM = IO.badpixelmask() all_solutions = [] cntr = 0 for plan in plans: p0 = plan[0].replace("'", "p") p1 = plan[1].replace("'", "p") suffix = "%s-%s" % (p0,p1) print "Handling plan %s" % suffix fname = "bsub_{0}_{1}_{2}.fits".format(maskname,band,suffix) EPS = IO.read_drpfits(maskname, fname, options) EPS[1] = np.ma.masked_array(EPS[1], theBPM, fill_value=0) fname = "var_{0}_{1}_{2}.fits".format(maskname, band, suffix) VAR = IO.read_drpfits(maskname, fname, options) VAR[1] = np.ma.masked_array(VAR[1], theBPM, fill_value=np.inf) fname = "itime_{0}_{1}_{2}.fits".format(maskname, band, suffix) ITIME = IO.read_drpfits(maskname, fname, options) ITIME[1] = np.ma.masked_array(ITIME[1], theBPM, fill_value=0) dats = EPS vars = VAR itimes = ITIME EPS[0]["ORIGFILE"] = fname tock = time.time() sols = range(len(edges)-1,-1,-1) shifts = all_shifts[cntr] cntr += 1 p = Pool() solutions = p.map(handle_rectification_helper, sols) #solutions = map(handle_rectification_helper, [15]) p.close() all_solutions.append(solutions) tick = time.time() print "-----> Mask took %i. Writing to disk." % (tick-tock) output = np.zeros((1, len(fidl))) snrs = np.zeros((1, len(fidl))) sdout= np.zeros((1, len(fidl))) itout= np.zeros((1, len(fidl))) # the barset [bs] is used for determining object position x, x, bs = IO.readmosfits(barset_file, options) for i_slit in xrange(len(solutions)): solution = all_solutions[0][i_slit] header = EPS[0].copy() obj = header['OBJECT'] target_name = bs.ssl[-(i_slit+1)]['Target_Name'] header['OBJECT'] = target_name pixel_dist = np.float(bs.ssl[-(i_slit+1)]['Target_to_center_of_slit_distance'])/0.18 pixel_dist -= solution['offset'] ll = solution["lambda"] header["wat0_001"] = "system=world" header["wat1_001"] = "wtype=linear" header["wat2_001"] = "wtype=linear" header["dispaxis"] = 1 header["dclog1"] = "Transform" header["dc-flag"] = 0 header["ctype1"] = "AWAV" header["cunit1"] = "Angstrom" header["crval1"] = ll[0] header["crval2"] = -solution["eps_img"].shape[0]/2 - pixel_dist header["crpix1"] = 1 header["crpix2"] = 1 header["cdelt1"] = 1 header["cdelt2"] = 1 header["cname1"] = "angstrom" header["cname2"] = "pixel" header["cd1_1"] = ll[1]-ll[0] header["cd1_2"] = 0 header["cd2_1"] = 0 header["cd2_2"] = 1 S = output.shape img = solution["eps_img"] std = solution["sd_img"] tms = solution["itime_img"] for i_solution in xrange(1,len(all_solutions)): print "Combining solution %i" %i_solution solution = all_solutions[i_solution][i_slit] img += solution["eps_img"] std += solution["sd_img"] tms += solution["itime_img"] output = np.append(output, img, 0) output = np.append(output, np.nan*np.zeros((3,S[1])), 0) snrs = np.append(snrs, img*tms/std, 0) snrs = np.append(snrs, np.nan*np.zeros((3,S[1])), 0) sdout = np.append(sdout, std, 0) sdout = np.append(sdout, np.nan*np.zeros((3,S[1])), 0) itout = np.append(itout, tms, 0) itout = np.append(itout, np.nan*np.zeros((3,S[1])), 0) header['bunit'] = ('electron/second', 'electron power') IO.writefits(img, maskname, "{0}_{1}_{2}_eps.fits".format(maskname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header['bunit'] = ('electron/second', 'sigma/itime') IO.writefits(std/tms, maskname, "{0}_{1}_{2}_sig.fits".format(maskname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header['bunit'] = ('second', 'exposure time') IO.writefits(tms, maskname, "{0}_{1}_{2}_itime.fits".format(maskname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header['bunit'] = ('', 'SNR') IO.writefits(img*tms/std, maskname, "{0}_{1}_{2}_snrs.fits".format(maskname, band, target_name), options, overwrite=True, header=header, lossy_compress=False) header = EPS[0].copy() header["wat0_001"] = "system=world" header["wat1_001"] = "wtype=linear" header["wat2_001"] = "wtype=linear" header["dispaxis"] = 1 header["dclog1"] = "Transform" header["dc-flag"] = 0 header["ctype1"] = "AWAV" header["cunit1"] = ("Angstrom", 'Start wavelength') header["crval1"] = ll[0] header["crval2"] = 1 header["crpix1"] = 1 header["crpix2"] = 1 header["cdelt1"] = 1 header["cdelt2"] = 1 header["cname1"] = "angstrom" header["cname2"] = "pixel" header["cd1_1"] = (ll[1]-ll[0], 'Angstrom/pixel') header["cd1_2"] = 0 header["cd2_1"] = 0 header["cd2_2"] = 1 header["bunit"] = "ELECTRONS/SECOND" IO.writefits(output, maskname, "{0}_{1}_eps.fits".format(maskname, band), options, overwrite=True, header=header, lossy_compress=False) header["bunit"] = "" IO.writefits(snrs, maskname, "{0}_{1}_snrs.fits".format(maskname, band), options, overwrite=True, header=header, lossy_compress=False) header["bunit"] = "ELECTRONS/SECOND" IO.writefits(sdout/itout, maskname, "{0}_{1}_sig.fits".format(maskname, band), options, overwrite=True, header=header, lossy_compress=False) header["bunit"] = "SECOND" IO.writefits(itout, maskname, "{0}_{1}_itime.fits".format(maskname, band), options, overwrite=True, header=header, lossy_compress=False)
def go(fname): global plot_arr print fname (header, data) = IO.readfits(fname) bs = CSU.Barset() bs.set_header(header) cntfit = 1 for i in range(1,8): continue pl.figure(i) pl.clf() first_time = True bars = [] means = [] sds = [] deltas = [] deltas_mm = [] poss = [] poss_mm = [] poss_y = [] request = [] qs = [] bars = range(1, 93) for bar in bars: pos = bs.get_bar_pix(bar) if bar % 8 == 0: print "%2.0i: (%7.2f, %7.2f)" % (bar, pos[0], pos[1]) width = 19 [[xslice, yslice],extent,ystart] = make_slice(pos,0,width,30) if not is_in_bounds(extent): fits = [0,0,0,0,0] [ff,ok] = [np.poly1d(0,0), []] means.append(fits) sds.append(fits) drop_this = True else: drop_this = False fits = [] ys = np.arange(-10,10, dtype=np.float32) for i in ys: tofit = data[ystart-i, xslice] y = median_tails(tofit) ps = Fit.do_fit(y, Fit.residual_single) fits.append(ps[0]) fits = np.array(fits) fits[:,1] += 1 # fit to the ridgeline [ff, ok] = fit_line_with_sigclip(ys, fits[:,1]) m = [np.mean(fits[:,i]) for i in range(5)] s = [np.std(fits[:,i]) for i in range(5)] means.append(m) sds.append(s) #if bar == 44: start_shell() #if bar == 43: start_shell() # End of measurement logic, PLOTS plot_arr = [12, 8, cntfit] cntfit += 1 if False: plot_stamps(data[yslice, xslice], ps[0]) y0 = median_tails(data[ystart, xslice]) plot_linefits(y0, ff) plot_ridgeline(fits, ok, ys, ff, bar) plot_widths(fits, ok, ys) # End of plots, BOOKEEPING slit_center_offset = pos[1] - ystart fc = ff(slit_center_offset) slit_center_pos = np.float(extent[0] + fc ) if drop_this: poss.append(NaN) poss_y.append(NaN) poss_mm.append(NaN) else: poss.append(slit_center_pos) poss_y.append(ystart) poss_mm.append(CSU.csu_pix_to_mm_poly(slit_center_pos, ystart)[0]) delta = np.float(slit_center_pos - pos[0]) if drop_this: deltas.append(NaN) deltas_mm.append(NaN) else: deltas.append(delta) b = CSU.csu_pix_to_mm_poly(slit_center_pos + delta, ystart)[0] deltas_mm.append(b - poss_mm[-1]) q = np.float(np.degrees(np.tan(ff(1)-ff(0)))) if drop_this: qs.append(NaN) qs.append(q) if False: #is_in_bounds(extent): pl.figure(2) pl.text(pos[0], pos[1], 'b%2.0i: w=%3.2f p=%5.2f q=%3.2f d=%1.3f' % (bar, np.mean(fits[:,4]), extent[0]+ff(0), q, delta), fontsize=11, family='monospace', horizontalalignment='center') means = np.array(means) f = lambda x: np.array(x).ravel() sds = f(sds) deltas = f(deltas) poss = f(poss) poss_y = f(poss_y) poss_mm = f(poss_mm) deltas_mm = f(deltas_mm) qs = f(qs) bars = f(bars) pl.draw() if False: [pf, ok] = fit_line_with_sigclip(bars, deltas) print "** Residual: %4.3f [pix]" % np.std(deltas[ok]-pf(bars[ok])) evens = range(0,92,2) odds = range(1,92,2) pl.plot(bars[evens], deltas[evens],'r*-') pl.plot(bars[odds], deltas[odds],'b*-') pl.title("Red: even, blue: odd") pl.xlabel("Bar #") pl.ylabel("Delta pixel (Measured - Predicted)") pl.plot(bars, pf(bars)) fout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".sav" print "saving" tosav = {"bars": bars, "request": bs.pos, "deltas_mm": deltas_mm, "poss": poss, "poss_mm": poss_mm, "deltas": deltas, "means": means, "qs": qs} scipy.io.savemat(fout, tosav) save_region(bs, "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".reg") print "saved" regout = "/users/npk/dropbox/mosfire/cooldown 9/csu_meas/" + fname.split("/")[-1] + ".meas.reg" pairs = np.array([poss,poss_y]).transpose() s = CSU.to_ds9_region(pairs, dash=0, color="blue", label=False) try: f = open(regout, "w") f.writelines(s) f.close() except: print "Couldn't write: %s" % regout return [tosav, bs]
def handle_background(filelist, wavename, maskname, band_name, options, shifts=None, plan=None, extension=None): ''' Perform difference imaging and subtract residual background. The plan looks something like: [['A', 'B']] In this case, the number of output files is equal to the length of the list (1). If you choose to use an ABA'B' pattern then the plan will be: [["A", "B"], ["A'", "B'"]] the background subtraction code will make and handle two files, "A-B" and "A'-B'". ''' global header, bs, edges, data, Var, itime, lam, sky_sub_out, sky_model_out, band band = band_name flatname = "pixelflat_2d_%s.fits" % band_name hdr, flat = IO.readfits("pixelflat_2d_%s.fits" % (band_name), options) if np.abs(np.median(flat) - 1) > 0.1: raise Exception("Flat seems poorly behaved.") ''' This next section of the code figures out the observing plan and then deals with the bookeeping of sending the plan to the background subtracter. ''' hdrs = [] epss = {} vars = {} bss = [] times = {} Nframes = [] i = 0 header = pf.Header() for i in xrange(len(filelist)): fl = filelist[i] files = IO.list_file_to_strings(fl) print "Combining" if shifts is None: shift = None else: shift = shifts[i] hdr, electron, var, bs, time, Nframe = imcombine(files, maskname, options, flat, outname="%s.fits" % (fl), shifts=shift, extension=extension) hdrs.append(hdr) header = merge_headers(header, hdr) epss[hdr['FRAMEID']] = electron/time vars[hdr['FRAMEID']] = var times[hdr['FRAMEID']] = time bss.append(bs) Nframes.append(Nframe) positions = {} i = 0 for h in hdrs: positions[h['FRAMEID']] = i i += 1 posnames = set(positions.keys()) if plan is None: plan = guess_plan_from_positions(posnames) num_outputs = len(plan) edges, meta = IO.load_edges(maskname, band, options) lam = IO.readfits(wavename, options) bs = bss[0] for i in xrange(num_outputs): posname0 = plan[i][0] posname1 = plan[i][1] print "Handling %s - %s" % (posname0, posname1) data = epss[posname0] - epss[posname1] Var = vars[posname0] + vars[posname1] itime = np.mean([times[posname0], times[posname1]], axis=0) p = Pool() solutions = p.map(background_subtract_helper, xrange(len(bs.ssl))) p.close() write_outputs(solutions, itime, header, maskname, band, plan[i], options)
def handle_flats(flatlist, maskname, band, options, extension=None, edgeThreshold=450, lampOffList=None, longslit=None): ''' handle_flats is the primary entry point to the Flats module. handle_flats takes a list of individual exposure FITS files and creates: 1. A CRR, dark subtracted, pixel-response flat file. 2. A set of polynomials that mark the edges of a slit Inputs: flatlist: maskname: The name of a mask band: A string indicating the bandceil Outputs: file {maskname}/flat_2d_{band}.fits -- pixel response flat file {maskname}/edges.np ''' tick = time.time() # Check bpos = np.ones(92) * -1 #Retrieve the list of files to use for flat creation. flatlist = IO.list_file_to_strings(flatlist) # Print the filenames to Standard-out for flat in flatlist: info(str(flat)) #Determine if flat files headers are in agreement for fname in flatlist: hdr, dat, bs = IO.readmosfits(fname, options, extension=extension) try: bs0 except: bs0 = bs if np.any(bs0.pos != bs.pos): print "bs0: " + str(bs0.pos) + " bs: " + str(bs.pos) error("Barset do not seem to match") raise Exception("Barsets do not seem to match") if hdr["filter"] != band: error("Filter name %s does not match header filter name " "%s in file %s" % (band, hdr["filter"], fname)) raise Exception("Filter name %s does not match header filter name " "%s in file %s" % (band, hdr["filter"], fname)) for i in xrange(len(bpos)): b = hdr["B{0:02d}POS".format(i + 1)] if bpos[i] == -1: bpos[i] = b else: if bpos[i] != b: error("Bar positions are not all the same in " "this set of flat files") raise Exception("Bar positions are not all the same in " "this set of flat files") bs = bs0 # Imcombine the lamps ON flats info("Attempting to combine previous files") combine(flatlist, maskname, band, options) # Imcombine the lamps OFF flats and subtract the off from the On sets if lampOffList != None: #Retrieve the list of files to use for flat creation. lampOffList = IO.list_file_to_strings(lampOffList) # Print the filenames to Standard-out for flat in lampOffList: info(str(flat)) print "Attempting to combine Lamps off data" combine(lampOffList, maskname, band, options, lampsOff=True) combine_off_on(maskname, band, options) debug("Combined '%s' to '%s'" % (flatlist, maskname)) info("Comgined to '%s'" % (maskname)) path = "combflat_2d_%s.fits" % band (header, data) = IO.readfits(path, use_bpm=True) info("Flat written to %s" % path) # Edge Trace if bs.long_slit: info("Long slit mode recognized") info("Central row position: " + str(longslit["row_position"])) info("Upper and lower limits: " + str(longslit["yrange"][0]) + " " + str(longslit["yrange"][1])) results = find_longslit_edges(data, header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit) elif bs.long2pos_slit: info("Long2pos mode recognized") results = find_long2pos_edges(data, header, bs, options, edgeThreshold=edgeThreshold, longslit=longslit) else: results = find_and_fit_edges(data, header, bs, options, edgeThreshold=edgeThreshold) results[-1]["maskname"] = maskname results[-1]["band"] = band np.save("slit-edges_{0}".format(band), results) save_ds9_edges(results, options) # Generate Flat out = "pixelflat_2d_%s.fits" % (band) if lampOffList != None: make_pixel_flat(data, results, options, out, flatlist, lampsOff=True) else: make_pixel_flat(data, results, options, out, flatlist, lampsOff=False) info("Pixel flat took {0:6.4} s".format(time.time() - tick))
def handle_flats(flatlist, maskname, band, options, extension=None): ''' handle_flats is the primary entry point to the Flats module. handle_flats takes a list of individual exposure FITS files and creates: 1. A CRR, dark subtracted, pixel-response flat file. 2. A set of polynomials that mark the edges of a slit Inputs: flatlist: Either a string of an input file or a list of file names maskname: The name of a mask band: A string indicating the bandceil Outputs: file {maskname}/flat_2d_{band}.fits -- pixel response flat file {maskname}/edges.np ''' tick = time.time() # Check bpos = np.ones(92) * -1 flatlist = IO.list_file_to_strings(flatlist) print flatlist for fname in flatlist: hdr, dat, bs = IO.readmosfits(fname, options, extension=extension) try: bs0 except: bs0 = bs if np.any(bs0.pos != bs.pos): raise Exception("Barsets do not seem to match") if hdr["filter"] != band: print ("Filter name %s does not match header filter name " "%s in file %s" % (band, hdr["filter"], fname)) for i in xrange(len(bpos)): b = hdr["B{0:02d}POS".format(i+1)] if bpos[i] == -1: bpos[i] = b else: if bpos[i] != b: raise Exception("Bar positions are not all the same in " "this set of flat files") bs = bs0 # Imcombine if True: print "Attempting to combine: ", flatlist combine(flatlist, maskname, band, options) print "Combined '%s' to '%s'" % (flatlist, maskname) path = "combflat_2d_%s.fits" % band (header, data) = IO.readfits(path, use_bpm=True) print "Flat written to %s" % path # Edge Trace results = find_and_fit_edges(data, header, bs, options) results[-1]["maskname"] = maskname results[-1]["band"] = band np.save("slit-edges_{0}".format(band), results) save_ds9_edges(results, options) # Generate Flat out = "pixelflat_2d_%s.fits" % (band) make_pixel_flat(data, results, options, out, flatlist) print "Pixel flat took {0:6.4} s".format(time.time()-tick)