def log_norm(image: np.ndarray, a=2000): return ImageNormalize(image, interval=ZScaleInterval(), stretch=LogStretch(a))
# if not os.path.isfile('{}/images/image_filtered_low_clean.fits'.format(odf_dir)): # fits_image = '{}/images/image_filtered_low.fits'.format(odf_dir) # else: # fits_image = '{}/images/image_filtered_low_clean.fits'.format(odf_dir) hdu = fits.open(fits_image) wcs = WCS(hdu[0].header) g2_kernel = Gaussian2DKernel(2) smoothed_data_g2 = convolve(hdu[0].data, g2_kernel, mask=np.logical_not(detmask.data)) * detmask.data fig = plt.figure(figsize=(10, 10), dpi=100) pp = 99.9 # colour cut percentage ax = fig.add_subplot(111, projection=wcs) #ax.set_title("Gaussian smoothed image") norm_xmm = ImageNormalize(smoothed_data_g2, interval=ManualInterval(vmin=0.01, vmax=100.0), stretch=LogStretch()) # norm_xmm = ImageNormalize(smoothed_data_g2,interval=PercentileInterval(pp), stretch=AsinhStretch()) ax.imshow(smoothed_data_g2, cmap=plt.cm.hot, norm=norm_xmm, origin='lower', interpolation='nearest') ax.xlabel = 'RA' ax.ylabel = 'Dec' ax.set_xlabel('RA') ax.set_ylabel('DEC') # # only show the last aperture # circle_sky = CircleSkyRegion(center=center, radius=r_end) pix_reg = circle_sky.to_pixel(wcs) pix_reg.plot(ax=ax, edgecolor='yellow') plt.savefig('/home/aaranda/tfm/results_v2/{}/{}/smoothed_g2_image_low.png'.format(target, obsid)) plt.close(fig)
rt = razmap(dfile, rbins, tbins, incl=disk.disk['HD143006']['incl'], PA=disk.disk['HD143006']['PA'], offx=disk.disk['HD143006']['dx'], offy=disk.disk['HD143006']['dy']) # Image setups im_bounds = (rt.dRA.max(), rt.dRA.min(), rt.dDEC.min(), rt.dDEC.max()) dRA_lims, dDEC_lims = [rs * rout, -rs * rout], [-rs * rout, rs * rout] rt_bounds = (rbins.min(), rbins.max(), tbins.min(), tbins.max()) rlims, tlims = [0, rbins.max()], [tbins.min(), tbins.max()] # intensity limits and stretch cnorm = ImageNormalize(vmin=vmin, vmax=vmax, stretch=LinearStretch()) # Residual emission map (sky-plane, Cartesian) plt.style.use('default') plt.rc('font', size=6) # set location in figure ax = fig.add_subplot(gs[0, 0]) # plot the residual image im = ax.imshow(1e6 * rt.image, origin='lower', cmap=cmap, extent=im_bounds, norm=cnorm, aspect='equal')
def kepmask(infile, frameno=100, maskfile='maskfile.txt', plotfile='kepmask.png', imin=None, imax=None, iscale='linear', cmap='bone', verbose=False, logfile='kepmask.log'): """ kepmask - plots, creates or edits custom target masks for target pixel files. The product from this task is a target mask definition file which can be used by kepextract to extract a light curve from target pixel data. This tool is a GUI interface for defining a pixel mask by moving a mouse over image pixels and selecting them by pressing the left-button of your mouse/keypad. Parameters ---------- infile : str The name of a target pixel file from the MAST Kepler archive, containing a standard mask definition image in the second data extension. frameno : int Frame number in the target pixel file. maskfile : str The name of an ASCII mask definition file. This is either the name of a file to be plotted, a file to be created, or a file to be edited. plotfile : str The name of a PNG plot file containing a record of the mask defined or uploaded by this task. imin : float or None Minimum intensity value (in electrons per cadence) for the image display. The default minimum intensity level is the median of the faintest 10% of pixels in the image. imax : float or None Maximum intensity value (in electrons per cadence) for the image display. The default maximum intensity level is the median of the brightest 10% of pixels in the image. iscale : str Type of intensity scaling for the image display. * linear * log * sqrt cmap : str Color intensity scheme for the image display. verbose : bool Print informative messages and warnings to the shell and logfile? logfile : str Name of the logfile containing error and warning messages. Examples -------- .. code-block:: bash $ kepmask ktwo202933888-c02_lpd-targ.fits.gz .. image:: ../_static/images/api/kepmask.png :align: center """ global pimg, zscale, zmin, zmax, xmin, xmax, ymin, ymax, quarter, norm global pxdim, pydim, kepmag, skygroup, season, channel global module, output, row, column, mfile, pfile global pkepid, pkepmag, pra, pdec, colmap global pxdim, pydim # input arguments zmin = imin; zmax = imax; zscale = iscale; colmap = cmap mfile = maskfile; pfile = plotfile # log the call hashline = '--------------------------------------------------------------' kepmsg.log(logfile, hashline, verbose) call = ('KEPMASK -- ' + ' infile={}'.format(infile) + ' maskfile={}'.format(mfile) + ' plotfile={}'.format(pfile) + ' frameno={}'.format(frameno) + ' imin={}'.format(imin) + ' imax={}'.format(imax) + ' iscale={}'.format(iscale) + ' cmap={}'.format(cmap) + ' verbose={}'.format(verbose) + ' logfile={}'.format(logfile)) kepmsg.log(logfile, call + '\n', verbose) kepmsg.clock('KEPMASK started at', logfile, verbose) # open TPF FITS file and check whether or not frameno exists try: tpf = pyfits.open(infile, mode='readonly') except: errmsg = ('ERROR -- KEPIO.OPENFITS: cannot open ' + infile + ' as a FITS file') kepmsg.err(logfile, errmsg, verbose) try: naxis2 = tpf['TARGETTABLES'].header['NAXIS2'] except: errmsg = ('ERROR -- KEPMASK: No NAXIS2 keyword in ' + infile + '[TARGETTABLES]') kepmsg.err(logfile, errmsg, verbose) if frameno > naxis2: errmsg = ('ERROR -- KEPMASK: frameno is too large. There are' ' {} rows in the table.'.format(naxis2)) kepmsg.err(logfile, errmsg, verbose) tpf.close() # read TPF data pixel image kepid, channel, skygroup, module, output, quarter, season, \ ra, dec, column, row, kepmag, xdim, ydim, pixels = \ kepio.readTPF(infile, 'FLUX', logfile, verbose) img = pixels[frameno] pkepid = copy(kepid) pra = copy(ra) pdec = copy(dec) pkepmag = copy(kepmag) pxdim = copy(xdim) pydim = copy(ydim) pimg = copy(img) # print target data print('') print(' KepID: {}'.format(kepid)) print(' RA (J2000): {}'.format(ra)) print('Dec (J2000): {}'.format(dec)) print(' KepMag: {}'.format(kepmag)) print(' SkyGroup: {}'.format(skygroup)) print(' Season: {}'.format(season)) print(' Channel: {}'.format(channel)) print(' Module: {}'.format(module)) print(' Output: {}'.format(output)) print('') # subimage of channel for plot ymin = copy(row) ymax = ymin + ydim xmin = copy(column) xmax = xmin + xdim # intensity scale if imin is None and imax is None: imin, imax = PercentileInterval(95.).get_limits(pimg) else: if imin is None: imin, _ = PercentileInterval(95.).get_limits(pimg) else: _, imax = PercentileInterval(95.).get_limits(pimg) if zscale == 'sqrt': norm = ImageNormalize(vmin=imin, vmax=imax, stretch=SqrtStretch()) elif zscale == 'linear': norm = ImageNormalize(vmin=imin, vmax=imax, stretch=LinearStretch()) elif zscale == 'log': norm = ImageNormalize(vmin=imin, vmax=imax, stretch=LogStretch()) zmin = copy(imin) zmax = copy(imax) # plot limits ymin = float(ymin) - 0.5 ymax = float(ymax) - 0.5 xmin = float(xmin) - 0.5 xmax = float(xmax) - 0.5 # plot style plt.rcParams['figure.dpi'] = 80 plt.figure(figsize=[10, 7]) global mask, aid, bid, cid, did, fid aid = plt.connect('button_press_event', clicker1) bid = plt.connect('button_press_event', clicker2) cid = plt.connect('button_press_event', clicker3) did = plt.connect('button_press_event', clicker4) fid = plt.connect('button_press_event', clicker6) redraw() plt.show()
backgroundcolor="white", transform=ax1[i].transAxes) ax1[i].plot(null, np.zeros_like(null), "r:", lw=2) ax1[i].errorbar(distance, fluxring[i], yerr=erb[i], fmt="ko-", lw=3, elinewidth=1, capsize=2, label=wav) # Plots SZ signal # normalises data with 'zscale' and stretches it by 'PowerStretch' as in ds9 norm = ImageNormalize(SZ[i], interval=zscale(), stretch=PowerStretch(1.3)) P1 = ax2[1, i].imshow(MCsim[i], norm=norm, aspect="auto") P2 = ax2[0, i].imshow(SZ[i], norm=norm, aspect="auto") cb = fig2.colorbar(P1, cax=cax[i], orientation="horizontal") cb.ax.xaxis.set_ticks_position("top") cb.set_label(r"$\mathrm{mJy/px}$", fontsize=10) cb.ax.xaxis.set_label_position("top") cb.ax.tick_params(labelsize=8) for j in range(len(ax2)): ax2[j, i].axis("off") t = ax2[j, i].text(0.69, 0.90, wav, color="w",
def create_index(filenames, directory, display=False, imagestretch='linear'): """ create index.html diagnostic root website for one pipeline process """ if display: print('create frame index table and frame images') logging.info('create frame index table and frame images') # obtain grating from first image file telescope, obsparam = CheckInstrument([filenames[0]]) refheader = fits.open(filenames[0], ignore_missing_end=True)[0].header grating = [refheader[obsparam['grating']]] del (refheader) html = "<H2>data directory: %s</H2>\n" % directory html += ("<H1>%s/%s-band - Diagnostic Output</H1>\n" + \ "%d frames total, see full pipeline " + \ "<A HREF=\"%s\">log</A> for more information\n") % \ (obsparam['telescope_instrument'], grating, len(filenames), '.diagnostics/' + _SP_conf.log_filename.split('.diagnostics/')[1]) ### create frame information table html += "<P><TABLE BORDER=\"1\">\n<TR>\n" html += "<TH>Idx</TH><TH>Filename</TH><TH>Date</TH>" + \ "<TH>Objectname</TH><TH>Grating</TH>" + \ "<TH>Airmass</TH><TH>Exptime (s)</TH>" + \ "<TH>Type</TH>\n</TR>\n" # fill table and create frames filename = filenames for idx, filename in enumerate(filenames): ### fill table hdulist = fits.open(filename, ignore_missing_end=True) header = hdulist[0].header # read out image binning mode binning = tb.get_binning(header, obsparam) #framefilename = _pp_conf.diagroot + '/' + filename + '.png' framefilename = '.diagnostics/' + filename + '.png' try: objectname = header[obsparam['object']] except KeyError: objectname = 'Unknown Target' html += ("<TR><TD>%d</TD><TD><A HREF=\"%s\">%s</A></TD>" + \ "<TD>%s</TD><TD>%s</TD>" + \ "<TD>%s</TD><TD>%4.2f</TD><TD>%.1f</TD>" + \ "<TD>%s</TD>\n</TR>\n") % \ (idx+1, framefilename, filename, header[obsparam['date_keyword']], objectname, header[obsparam['grating']], float(header[obsparam['airmass']]), float(header[obsparam['exptime']]), header[obsparam['obstype']]) ### create frame image imgdat = hdulist[0].data # clip extreme values to prevent crash of imresize imgdat = np.clip(imgdat, np.percentile(imgdat, 1), np.percentile(imgdat, 99)) imgdat = imresize(imgdat, min(1., 1000. / np.max(imgdat.shape)), interp='nearest') #resize image larger than 1000px on one side norm = ImageNormalize(imgdat, interval=ZScaleInterval(), stretch={ 'linear': LinearStretch(), 'log': LogStretch() }[imagestretch]) plt.figure(figsize=(5, 5)) img = plt.imshow(imgdat, cmap='gray', norm=norm, origin='lower') # remove axes plt.axis('off') img.axes.get_xaxis().set_visible(False) img.axes.get_yaxis().set_visible(False) plt.savefig(framefilename, format='png', bbox_inches='tight', pad_inches=0, dpi=200) plt.close() hdulist.close() del (imgdat) html += '</TABLE>\n' create_website(_SP_conf.index_filename, html) ### add to summary website, if requested # if _pp_conf.use_diagnostics_summary: # add_to_summary(header[obsparam['object']], filtername, # len(filenames)) return None
def add_Extract(filenames_im, filenames_spec, html_file, imagestretch='linear'): """ add pre-processing log to website """ # update index.html html = '<H2>Extract list</H2>\n' html += '%d images have been processed' % \ (len(filenames_im)) html += "<P><TABLE BORDER=\"1\">\n<TR>\n" html += "<TH>Idx</TH><TH>Filename</TH> <TH>Images</TH></TR>\n" for idx, filename in enumerate(filenames_im): ### fill table spec = np.loadtxt(filenames_spec[idx]) plt.figure() plt.plot(spec) plt.xlabel('pixels') plt.ylabel('Relative reflectance') specfilename = 'diagnostics/' + filenames_spec[idx] + '.png' plt.savefig(specfilename) plt.close() hdulist = fits.open(filename, ignore_missing_end=True) if not os.path.isdir('diagnostics'): os.mkdir('diagnostics') #framefilename = _SP_conf.diagroot + '/' + filename + '.png' framefilename = 'diagnostics/' + filename + '.png' html += ("<TR><TD>%d</TD><TD><A HREF=\"%s\">%s</A></TD> <TH> <IMG SRC=\"%s\"> </TH> <TH> <IMG SRC=\"%s\"> </TH></TR>\n") % \ (idx+1, framefilename, filename,framefilename,specfilename) ### create frame image imgdat = hdulist[0].data # clip extreme values to prevent crash of imresize imgdat = np.clip(imgdat, np.percentile(imgdat, 1), np.percentile(imgdat, 99)) imgdat = imresize(imgdat, min(1., 1000. / np.max(imgdat.shape)), interp='nearest') #resize image larger than 1000px on one side norm = ImageNormalize(imgdat, interval=ZScaleInterval(), stretch={ 'linear': LinearStretch(), 'log': LogStretch() }[imagestretch]) plt.figure(figsize=(5, 5)) img = plt.imshow(imgdat, cmap='gray', norm=norm, origin='lower') # remove axes plt.axis('off') img.axes.get_xaxis().set_visible(False) img.axes.get_yaxis().set_visible(False) plt.savefig(framefilename, format='png', bbox_inches='tight', pad_inches=0, dpi=200) plt.close() hdulist.close() del (imgdat) html += '</TABLE>\n' append_website(html_file, html, replace_below="<H2>Extract list</H2>\n")
def kde2D_plot(self, parameter1, parameter2, normtype='log', interval=None, xlim=None, ylim=None, gridsize=100): """Generate a 2D KDE for the given parameters. Parameters ---------- parameter1 : `numpy.array` X-axis variable parameter2 : `numpy.array` Y-axis variable normtype : {'log', 'linear', 'sqrt'} Normalization type to apply to the data interval : tuple Limits of the interval to use when computing the image scaling xlim : tuple X-limits to use for the plot and the KDE grid ylim : tuple Y-limits to use for the plot and the KDE grid gridsize : int Step-size for the grid Returns ------- fig : :py:class:`matplotlib.figure.Figure` ax : :py:class:`matplotlib.axes.Axes` surface : numpy.array The KDE surface plot """ data = np.vstack([parameter1, parameter2]) if xlim is None: xlim = (np.min(parameter1), np.max(parameter1)) if ylim is None: ylim = (np.min(parameter2), np.max(parameter2)) # Generate a grid to compute the KDE over xgrid = np.linspace(xlim[0], xlim[1], gridsize) ygrid = np.linspace(ylim[0], ylim[1], gridsize) kde = gaussian_kde(data) Xgrid, Ygrid = np.meshgrid(xgrid, ygrid) surface = kde.evaluate(np.vstack([Xgrid.ravel(), Ygrid.ravel()])) if isinstance(interval, tuple): Interval = ManualInterval(vmin=interval[0], vmax=interval[1]) else: Interval = ZScaleInterval() norm = ImageNormalize(surface, stretch=self.image_norms[normtype], interval=Interval) fig, ax = self.mk_fig(nrows=1, ncols=1) ax.imshow(surface.reshape(Xgrid.shape), norm=norm, cmap='gray', origin='lower', aspect='auto', extent=[xgrid.min(), xgrid.max(), ygrid.min(), ygrid.max()]) return fig, ax, surface
def plot_hst_loc(self, i=5, df=None, title='', thresh=5, fout='', min_exptime=800, key='start', save=False, orbital_path1=None, orbital_path2=None): self.fig = plt.figure(figsize=(8, 6)) # Get the model for the SAA self.map = Basemap(projection='cyl') self._draw_map() df = df[df.integration_time.gt(min_exptime)] df.sort_values(by='incident_cr_rate', inplace=True) cbar_bounds = [0, 20, 40, 60, 80, 100, 120, 140, 160] sci_cmap = plt.cm.gray custom_norm = colors.BoundaryNorm(boundaries=cbar_bounds, ncolors=sci_cmap.N) # Generate an SAA contour saa = [list(t) for t in zip(*costools.saamodel.saaModel(i))] # Ensure the polygon representing the SAA is a closed curve by adding # the starting points to the end of the list of lat/lon coords saa[0].append(saa[0][0]) saa[1].append(saa[1][0]) self.map.plot(saa[1], saa[0], c='k', latlon=True, label='SAA contour {}'.format(i)) # df = self.perform_SAA_cut(df=df, key=key) if df is None: lat, lon, rate = self.data_df['latitude_{}'.format(key)], \ self.data_df['longitude_{}'.format(key)], \ self.data_df['incident_cr_rate'] else: #df = df[df['integration_time'] > 800] lat, lon, rate = df['latitude_{}'.format(key)], \ df['longitude_{}'.format(key)], \ df['incident_cr_rate'] LOG.info('{} {} {}'.format(len(lat), len(lon), len(rate))) # lat1, lon1, rate1 = lat[rate >0], lon[rate >0], rate[rate>0] # LOG.info('{} {} {}'.format(len(lat), len(lon), len(rate))) # median = np.median(rate) # std = np.std(rate) mean, median, std = sigma_clipped_stats(rate, sigma_lower=3, sigma_upper=3) LOG.info('{} +\- {}'.format(median, std)) norm = ImageNormalize(rate, stretch=LinearStretch(), vmin=mean - thresh * std, vmax=mean + thresh * std) cbar_below_mean = [mean - (i + 1) * std for i in range(thresh)] cbar_above_mean = [mean + (i + 1) * std for i in range(thresh)] cbar_bounds = cbar_below_mean + [mean] + cbar_above_mean print(cbar_bounds) cbar_bounds.sort() sci_cmap = plt.cm.viridis custom_norm = colors.BoundaryNorm(boundaries=cbar_bounds, ncolors=sci_cmap.N) scat = self.map.scatter(lon.values, lat.values, marker='o', s=5, latlon=True, c=rate, alpha=0.15, norm=custom_norm, cmap='viridis') #im = self.map.contourf(lon_grid, lat_grid, rate, norm=norm, cmap='viridis') ax = plt.gca() ax.set_title(title) # Plot the path of HST #self.map.plot( # orbital_path1.metadata['longitude'], # orbital_path1.metadata['latitude'],lw=1.25, # label=f'Int. Time: {1000:.1f}s', color='k', ls='-' #) if orbital_path2 is not None: self.map.scatter(orbital_path2.metadata['longitude'][::4][1:], orbital_path2.metadata['latitude'][::4][1:], c='k', s=20, label='285 seccond interval') if orbital_path1 is not None: self.map.plot(orbital_path2.metadata['longitude'], orbital_path2.metadata['latitude'], label=f'Orbital Path Over {2000:.0f} seconds', color='k', ls='--', lw=1.25) ax1_legend = ax.legend(loc='upper right', ncol=1, labelspacing=0.2, columnspacing=0.5, edgecolor='k') # for i in range(len(ax1_legend.legendHandles)): # ax1_legend.legendHandles[i]._sizes = [30] #cbar_tick_labels = [f'<x>-{thresh}$\sigma$', '<x>', f'<x>+{thresh}$\sigma$'] #cbar_ticks = [mean - thresh*std,mean, mean + thresh*std] cbar_ticks = cbar_bounds cax = self.fig.add_axes([0.1, 0.1, 0.8, 0.05]) cbar = self.fig.colorbar(scat, cax=cax, ticks=cbar_ticks, orientation='horizontal') cbar.set_alpha(1) cbar.draw_all() cbar_tick_labels = [f'<x>-{i}$\sigma$' for i in [5, 4, 3, 2, 1]] + [ '<x>' ] + [f'<x>+{i}$\sigma$' for i in [1, 2, 3, 4, 5]] cbar.ax.set_xticklabels(cbar_tick_labels, horizontalalignment='right', rotation=30) cbar.set_label('CR Flux [CR/s/$cm^2$]', fontsize=10) # cbar.ax.set_xticklabels(cbar.ax.get_xticklabels(), # fontweight='medium',fontsize=8) if save: if not fout: fout = 'lat_lon_{}.png'.format(key) self.fig.savefig(fout, format='png', bbox_inches='tight', dpi=350, transparent=False) plt.show() return self.fig
def get_finding_chart( source_ra, source_dec, source_name, image_source='ps1', output_format='pdf', imsize=3.0, tick_offset=0.02, tick_length=0.03, fallback_image_source='dss', zscale_contrast=0.045, zscale_krej=2.5, extra_display_string="", **offset_star_kwargs, ): """Create a finder chart suitable for spectroscopic observations of the source Parameters ---------- source_ra : float Right ascension (J2000) of the source source_dec : float Declination (J2000) of the source source_name : str Name of the source image_source : {'desi', 'dss', 'ztfref', 'ps1'}, optional Survey where the image comes from "desi", "dss", "ztfref", "ps1" defaults to "ps1" output_format : str, optional "pdf" of "png" -- determines the format of the returned finder imsize : float, optional Requested image size (on a size) in arcmin. Should be between 2-15. tick_offset : float, optional How far off the each source should the tick mark be made? (in arcsec) tick_length : float, optional How long should the tick mark be made? (in arcsec) fallback_image_source : str, optional Where what `image_source` should we fall back to if the one requested fails zscale_contrast : float, optional Contrast parameter for the ZScale interval zscale_krej : float, optional Krej parameter for the Zscale interval extra_display_string : str, optional What else to show for the source itself in the chart (e.g. proper motion) **offset_star_kwargs : dict, optional Other parameters passed to `get_nearby_offset_stars` Returns ------- dict success : bool Whether the request was successful or not, returning a sensible error in 'reason' name : str suggested filename based on `source_name` and `output_format` data : str binary encoded data for the image (to be streamed) reason : str If not successful, a reason is returned. """ if (imsize < 2.0) or (imsize > 15): return { 'success': False, 'reason': 'Requested `imsize` out of range', 'data': '', 'name': '', } if image_source not in source_image_parameters: return { 'success': False, 'reason': f'image source {image_source} not in list', 'data': '', 'name': '', } matplotlib.use("Agg") fig = plt.figure(figsize=(11, 8.5), constrained_layout=False) widths = [2.6, 1] heights = [2.6, 1] spec = fig.add_gridspec( ncols=2, nrows=2, width_ratios=widths, height_ratios=heights, left=0.05, right=0.95, ) # how wide on the side will the image be? 256 as default npixels = source_image_parameters[image_source].get("npixels", 256) # set the pixelscale in arcsec (typically about 1 arcsec/pixel) pixscale = 60 * imsize / npixels hdu = fits_image(source_ra, source_dec, imsize=imsize, image_source=image_source) # skeleton WCS - this is the field that the user requested wcs = WCS(naxis=2) # set the headers of the WCS. # The center of the image is the reference point (source_ra, source_dec): wcs.wcs.crpix = [npixels / 2, npixels / 2] wcs.wcs.crval = [source_ra, source_dec] # create the pixel scale and orientation North up, East left # pixelscale is in degrees, established in the tangent plane # to the reference point wcs.wcs.cd = np.array([[-pixscale / 3600, 0], [0, pixscale / 3600]]) wcs.wcs.ctype = ["RA---TAN", "DEC--TAN"] fallback = True if hdu is not None: im = hdu.data # replace the nans with medians im[np.isnan(im)] = np.nanmedian(im) # Fix the header keyword for the input system, if needed hdr = hdu.header if 'RADECSYS' in hdr: hdr.set('RADESYSa', hdr['RADECSYS'], before='RADECSYS') del hdr['RADECSYS'] if source_image_parameters[image_source].get("reproject", False): # project image to the skeleton WCS solution log("Reprojecting image to requested position and orientation") im, _ = reproject_adaptive(hdu, wcs, shape_out=(npixels, npixels)) else: wcs = WCS(hdu.header) if source_image_parameters[image_source].get("smooth", False): im = gaussian_filter( hdu.data, source_image_parameters[image_source]["smooth"] / pixscale) cent = int(npixels / 2) width = int(0.05 * npixels) test_slice = slice(cent - width, cent + width) all_nans = np.isnan(im[test_slice, test_slice].flatten()).all() all_zeros = (im[test_slice, test_slice].flatten() == 0).all() if not (all_zeros or all_nans): percents = np.nanpercentile(im.flatten(), [10, 99.0]) vmin = percents[0] vmax = percents[1] interval = ZScaleInterval( nsamples=int(0.1 * (im.shape[0] * im.shape[1])), contrast=zscale_contrast, krej=zscale_krej, ) norm = ImageNormalize(im, vmin=vmin, vmax=vmax, interval=interval) watermark = source_image_parameters[image_source]["str"] fallback = False if hdu is None or fallback: # if we got back a blank image, try to fallback on another survey # and return the results from that call if fallback_image_source is not None: if fallback_image_source != image_source: log(f"Falling back on image source {fallback_image_source}") return get_finding_chart( source_ra, source_dec, source_name, image_source=fallback_image_source, output_format=output_format, imsize=imsize, tick_offset=tick_offset, tick_length=tick_length, fallback_image_source=None, **offset_star_kwargs, ) # we dont have an image here, so let's create a dummy one # so we can still plot im = np.zeros((npixels, npixels)) watermark = None vmin = 0 vmax = 0 norm = ImageNormalize(im, vmin=vmin, vmax=vmax) # add the images in the top left corner ax = fig.add_subplot(spec[0, 0], projection=wcs) ax_text = fig.add_subplot(spec[0, 1]) ax_text.axis('off') ax_starlist = fig.add_subplot(spec[1, 0:]) ax_starlist.axis('off') ax.imshow(im, origin='lower', norm=norm, cmap='gray_r') ax.set_autoscale_on(False) ax.grid(color='white', ls='dotted') ax.set_xlabel(r'$\alpha$ (J2000)', fontsize='large') ax.set_ylabel(r'$\delta$ (J2000)', fontsize='large') obstime = offset_star_kwargs.get("obstime", datetime.datetime.utcnow().isoformat()) ax.set_title( f'{source_name} Finder (for {obstime.split("T")[0]})', fontsize='large', fontweight='bold', ) star_list, _, _, _, used_ztfref = get_nearby_offset_stars( source_ra, source_dec, source_name, **offset_star_kwargs) if not isinstance(star_list, list) or len(star_list) == 0: return { 'success': False, 'reason': 'failure to get star list', 'data': '', 'name': '', } ncolors = len(star_list) if star_list[0]['str'].startswith("!Data"): ncolors -= 1 colors = sns.color_palette("colorblind", ncolors) start_text = [-0.45, 0.99] origin = "GaiaEDR3" if not used_ztfref else "ZTFref" starlist_str = ( f"# Note: {origin} used for offset star positions\n" "# Note: spacing in starlist many not copy/paste correctly in PDF\n" + "# you can get starlist directly from" + f" /api/sources/{source_name}/offsets?" + f"facility={offset_star_kwargs.get('facility', 'Keck')}\n" + "\n".join([x["str"] for x in star_list])) # add the starlist ax_starlist.text( 0, 0.50, starlist_str, fontsize="x-small", family='monospace', transform=ax_starlist.transAxes, ) # add the watermark for the survey props = dict(boxstyle='round', facecolor='gray', alpha=0.7) if watermark is not None: ax.text( 0.035, 0.035, watermark, horizontalalignment='left', verticalalignment='center', transform=ax.transAxes, fontsize='medium', fontweight='bold', color="yellow", alpha=0.5, bbox=props, ) date_obs = hdr.get('DATE-OBS') if not date_obs: mjd_obs = hdr.get('MJD-OBS') if mjd_obs: date_obs = Time(f"{mjd_obs}", format='mjd').to_value('fits', subfmt='date_hms') if date_obs: ax.text( 0.95, 0.95, f'image date {date_obs.split("T")[0]}', horizontalalignment='right', verticalalignment='center', transform=ax.transAxes, fontsize='small', color="yellow", alpha=0.5, bbox=props, ) ax.text( 0.95, 0.035, f"{imsize}\u2032 \u00D7 {imsize}\u2032", # size'x size' horizontalalignment='right', verticalalignment='center', transform=ax.transAxes, fontsize='medium', fontweight='bold', color="yellow", alpha=0.5, bbox=props, ) # compass rose # rose_center_pixel = ax.transAxes.transform((0.04, 0.95)) rose_center = pixel_to_skycoord(int(npixels * 0.1), int(npixels * 0.9), wcs) props = dict(boxstyle='round', facecolor='gray', alpha=0.5) for ang, label, off in [(0, "N", 0.01), (90, "E", 0.03)]: position_angle = ang * u.deg separation = (0.05 * imsize * 60) * u.arcsec # 5% p2 = rose_center.directional_offset_by(position_angle, separation) ax.plot( [rose_center.ra.value, p2.ra.value], [rose_center.dec.value, p2.dec.value], transform=ax.get_transform('world'), color="gold", linewidth=2, ) # label N and E position_angle = (ang + 15) * u.deg separation = ((0.05 + off) * imsize * 60) * u.arcsec p2 = rose_center.directional_offset_by(position_angle, separation) ax.text( p2.ra.value, p2.dec.value, label, color="gold", transform=ax.get_transform('world'), fontsize='large', fontweight='bold', ) # account for Shane header if star_list[0]['str'].startswith("!Data"): star_list = star_list[1:] for i, star in enumerate(star_list): c1 = SkyCoord(star["ra"] * u.deg, star["dec"] * u.deg, frame='icrs') # mark up the right side of the page with position and offset info name_title = star["name"] if star.get("mag") is not None: name_title += f" {star.get('mag'):.2f} mag" ax_text.text( start_text[0], start_text[1] - (i * 1.1) / ncolors, name_title, ha='left', va='top', fontsize='large', fontweight='bold', transform=ax_text.transAxes, color=colors[i], ) source_text = f" {star['ra']:.5f} {star['dec']:.5f}\n" source_text += f" {c1.to_string('hmsdms', precision=2)}\n" if i == 0 and extra_display_string != "": source_text += f" {extra_display_string}\n" if ((star.get("dras") is not None) and (star.get("ddecs") is not None) and (star.get("pa") is not None)): source_text += f' {star.get("dras")} {star.get("ddecs")} (PA={star.get("pa"):<0.02f}°)' ax_text.text( start_text[0], start_text[1] - (i * 1.1) / ncolors - 0.06, source_text, ha='left', va='top', fontsize='large', transform=ax_text.transAxes, color=colors[i], ) # work on making marks where the stars are for ang in [0, 90]: # for the source itself (i=0), change the angle of the lines in # case the offset star is the same as the source itself position_angle = ang * u.deg if i != 0 else (ang + 225) * u.deg separation = (tick_offset * imsize * 60) * u.arcsec p1 = c1.directional_offset_by(position_angle, separation) separation = (tick_offset + tick_length) * imsize * 60 * u.arcsec p2 = c1.directional_offset_by(position_angle, separation) ax.plot( [p1.ra.value, p2.ra.value], [p1.dec.value, p2.dec.value], transform=ax.get_transform('world'), color=colors[i], linewidth=3 if imsize <= 4 else 2, alpha=0.8, ) if star["name"].find("_o") != -1: # this is an offset star text = star["name"].split("_o")[-1] position_angle = 14 * u.deg separation = (tick_offset + tick_length * 1.6) * imsize * 60 * u.arcsec p1 = c1.directional_offset_by(position_angle, separation) ax.text( p1.ra.value, p1.dec.value, text, color=colors[i], transform=ax.get_transform('world'), fontsize='large', fontweight='bold', ) buf = io.BytesIO() fig.savefig(buf, format=output_format) plt.close(fig) buf.seek(0) return { "success": True, "name": f"finder_{source_name}.{output_format}", "data": buf.read(), "reason": "", }
def pathtest(step_input_filename, reffile, comparison_filename, writefile=True, show_figs=False, save_figs=True, threshold_diff=1.0e-7, debug=False): """ This function calculates the difference between the pipeline and calculated pathloss values. Args: step_input_filename: str, full path name of sourcetype output fits file reffile: str, path to the pathloss IFU reference fits file comparison_filename: str, path to comparison pipeline pathloss file writefile: boolean, if True writes calculated flat and difference image fits files show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots threshold_diff: float, threshold diff between pipeline and ESA file debug: boolean, if true print statements will show on-screen Returns: - 1 plot, if told to save and/or show them. - median_diff: Boolean, True if smaller or equal to threshold. - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer pathtest_start_time = time.time() # get info from the rate file header msg = 'step_input_filename=' + step_input_filename print(msg) log_msgs.append(msg) exptype = fits.getval(step_input_filename, "EXP_TYPE", 0) grat = fits.getval(step_input_filename, "GRATING", 0) filt = fits.getval(step_input_filename, "FILTER", 0) msg = "pathloss file: Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype print(msg) log_msgs.append(msg) msg = "Using reference file: " + reffile print(msg) log_msgs.append(msg) is_point_source = True if writefile: # create the fits list to hold the calculated pathloss values for each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create fits list to hold pipeline-calculated difference values hdu0 = fits.PrimaryHDU() compfile = fits.HDUList() compfile.append(hdu0) # list to determine if pytest is passed or not total_test_result = [] # get all the science extensions if is_point_source: ext = 1 # for all PS IFU # get files print('Checking files exist & obtaining datamodels. Takes a few mins...') if os.path.isfile(comparison_filename): if debug: print('Comparison file does exist.') else: result_msg = 'Comparison file does NOT exist. Skipping pathloss test.' print(result_msg) log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the comparison data model ifu_pipe_model = datamodels.open(comparison_filename) if debug: print('got comparison datamodel!') if os.path.isfile(step_input_filename): if debug: print('Input file does exist.') else: result_msg = 'Input file does NOT exist. Skipping pathloss test.' log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the input data model ifu_input_model = datamodels.open(step_input_filename) if debug: print('got input datamodel!') # get slices (instead of using .slit) pl_ifu_slits = nirspec.nrs_ifu_wcs(ifu_input_model) print("got input slices") plcor_ref_ext = fits.getdata(reffile, ext) hdul = fits.open(reffile) plcor_ref = hdul[1].data w = wcs.WCS(hdul[1].header) w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref.shape[1], :plcor_ref. shape[2]] slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0) # these are full 2048 * 2048 files: previous_sci = fits.getdata(step_input_filename, "SCI") comp_sci = fits.getdata(comparison_filename, "SCI") pathloss_divided = comp_sci / previous_sci # Can manually test correction at nonzero point # slit_x = -0.3 # slit_y = 0.3 # if debug: # print("""WARNING: Using manually set slit_x and slit_y! # The pipeline correction will not use manually set values # and thus the residuals will change""") # set up generals for all plots font = {'weight': 'normal', 'size': 10} matplotlib.rc('font', **font) # loop through the slices msg = " Looping through the slices... " print(msg) log_msgs.append(msg) slit_list = np.ndarray.tolist(np.arange(0, 30)) for slit, slit_num in zip(pl_ifu_slits, slit_list): print("working with slice {}".format(slit_num)) x, y = wcstools.grid_from_bounding_box(slit.bounding_box, step=(1, 1)) ra, dec, wave = slit(x, y) slit_x = 0 # This assumption is made for IFU sources only slit_y = 0 if debug: print("slit_x, slit_y", slit_x, slit_y) correction_array = np.array([]) lambda_array = np.array([]) wave_sci = wave * 10**(-6) # microns --> meters wave_sci_flat = wave_sci.reshape(wave_sci.size) wave_ref_flat = wave_ref.reshape(wave_ref.size) ref_xy = np.column_stack((slitx_ref.reshape(slitx_ref.size), slity_ref.reshape(slitx_ref.size))) # loop through slices in lambda from reference file shape = 0 for lambda_val in wave_ref_flat: # loop through every lambda value # flattened so that looping works smoothly shape = shape + 1 index = np.where(wave_ref[:, 0, 0] == lambda_val) # index of closest lambda value in reffile to given sci lambda # took index of only the first slice of wave_ref because # the others were repetitive & we got extra indices # take slice where lambda=index: plcor_slice = plcor_ref_ext[index[0][0]].reshape( plcor_ref_ext[index[0][0]].size) # do 2d interpolation to get a single correction factor for each slice corr_val = scipy.interpolate.griddata(ref_xy[:plcor_slice.size], plcor_slice, np.asarray([slit_x, slit_y]), method='linear') # append values from loop to create a vector of correction factors correction_array = np.append(correction_array, corr_val[0]) # map to array with corresponding lambda lambda_array = np.append(lambda_array, lambda_val) # get correction value for each pixel corr_vals = np.interp(wave_sci_flat, lambda_array, correction_array) corr_vals = corr_vals.reshape(wave_sci.shape) box = slit.bounding_box small_y = box[0][0] big_y = box[0][1] small_x = box[1][0] big_x = box[1][1] left = int(math.trunc(small_x)) right = int(math.ceil(big_x)) bottom = int(math.trunc(small_y)) top = int(math.ceil(big_y)) full_cut2slice = previous_sci[left:right, bottom:top] print("shapes:", full_cut2slice.shape, corr_vals.shape) if full_cut2slice.shape != corr_vals.shape: value = 0 while (((full_cut2slice.shape[0] - corr_vals.shape[0]) != 0) or ((full_cut2slice.shape[1] - corr_vals.shape[1]) != 0)) and \ value < 7: # can delete second criteria once all pass if value == 6: print("WARNING: may be in infinite loop!") x_amount_off = full_cut2slice.shape[0] - corr_vals.shape[0] if x_amount_off >= 1: if x_amount_off % 2 == 0: # need to add two values, so do one on each side right = right - 1 left = left + 1 print("ALTERED SHAPE OF SLICE: V1") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: # just add one value left = left + 1 print("ALTERED SHAPE OF SLICE: V2") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) elif x_amount_off <= -1: if x_amount_off % 2 == 0: right = right + 1 left = left - 1 print("ALTERED SHAPE OF SLICE: V3") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: left = left - 1 print("ALTERED SHAPE OF SLICE: V4") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) y_amount_off = full_cut2slice.shape[1] - corr_vals.shape[1] if y_amount_off >= 1: if y_amount_off % 2 == 0: bottom = bottom - 1 top = top + 1 print("ALTERED SHAPE OF SLICE: V5") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: bottom = bottom + 1 print("ALTERED SHAPE OF SLICE: V6") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) elif y_amount_off <= -1: if y_amount_off % 2 == 0: top = top + 1 bottom = bottom - 1 print("ALTERED SHAPE OF SLICE: V7") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) else: bottom = bottom - 1 print("ALTERED SHAPE OF SLICE: V8") value = value + 1 print("left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) full_cut2slice = previous_sci[left:right, bottom:top] print("final left {}, right {}, top {}, bottom {}".format( left, right, top, bottom)) print("NEW SHAPE OF SLICE: {} and corr_vals.shape: {}".format( full_cut2slice.shape, corr_vals.shape)) if full_cut2slice.shape != corr_vals.shape: print("shapes did not match! full_cut2slice: {}, corr_vals {}". format(full_cut2slice.shape, corr_vals.shape)) continue corrected_array = full_cut2slice / corr_vals pipe_correction = pathloss_divided[left:right, bottom:top] if pipe_correction.shape != corr_vals.shape: print("shapes did not match! pipe_correction: {}, corr_vals {}". format(pipe_correction.shape, corr_vals.shape)) continue prev_sci_slit = previous_sci[left:right, bottom:top] if prev_sci_slit.shape != corr_vals.shape: print( "shapes did not match! prev_sci_slit: {}, corr_vals {}".format( prev_sci_slit.shape, corr_vals.shape)) continue comp_sci_slit = comp_sci[left:right, bottom:top] if comp_sci_slit.shape != corr_vals.shape: print( "shapes did not match! comp_sci_slit: {}, corr_vals {}".format( comp_sci_slit.shape, corr_vals.shape)) continue # Plots: step_input_filepath = step_input_filename.replace(".fits", "") # my correction values fig = plt.figure(figsize=(15, 15)) plt.subplot(221) norm = ImageNormalize(corr_vals) plt.imshow(corr_vals, vmin=0.999995, vmax=1.000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Calculated Correction') plt.colorbar() # pipe correction plt.subplot(222) norm = ImageNormalize(pipe_correction) plt.imshow(pipe_correction, vmin=0.999995, vmax=1.000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Pipe Correction') plt.colorbar() # residuals (pipe correction - my correction) if pipe_correction.shape == corr_vals.shape: corr_residuals = pipe_correction - corr_vals plt.subplot(223) norm = ImageNormalize(corr_residuals) plt.imshow(corr_residuals, vmin=-0.000000005, vmax=0.000000005, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Correction residuals') plt.colorbar() # my science data after pathloss plt.subplot(224) norm = ImageNormalize(corrected_array) plt.imshow(corrected_array, vmin=0, vmax=300, aspect=10.0, origin='lower', cmap='viridis') plt.title('My slit science data after pathloss') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.colorbar() fig.suptitle("IFU PS Pathloss Correction Testing" + str(corrected_array.shape)) if show_figs: plt.show() if save_figs: plt_name = step_input_filepath + "_Pathloss_test_slitlet_IFU_PS_" + str( slit_num) + ".png" plt.savefig(plt_name) print('Figure saved as: ', plt_name) elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." if debug: print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." if debug: print(msg) log_msgs.append(msg) plt.clf() # create fits file to hold the calculated pathloss for each slit if writefile: msg = "Saving the fits files with the calculated pathloss for each slit..." print(msg) log_msgs.append(msg) # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(corr_vals, name=str(slit_num)) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values compfile_ext = fits.ImageHDU(corr_residuals, name=str(slit_num)) compfile.append(compfile_ext) if corr_residuals[~np.isnan(corr_residuals)].size == 0: msg1 = " * Unable to calculate statistics because difference array has all values as NaN. " \ "Test will be set to FAILED." print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: msg = "Calculating statistics... " print(msg) log_msgs.append(msg) corr_residuals = corr_residuals[ np.where((corr_residuals != 999.0) & (corr_residuals < 0.1) & (corr_residuals > -0.1))] # ignore outliers if corr_residuals.size == 0: msg1 = """ * Unable to calculate statistics because difference array has all outlier values. Test will be set to FAILED.""" print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: stats_and_strings = auxfunc.print_stats(corr_residuals, "Difference", float(threshold_diff), absolute=True) stats, stats_print_strings = stats_and_strings corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats for msg in stats_print_strings: log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(corr_residuals_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) total_test_result.append(test_result) if writefile: outfile_name = step_input_filename.replace("srctype", "_calcuated_pathloss") compfile_name = step_input_filename.replace("srctype", "_comparison_pathloss") # create the fits list to hold the calculated pathloss values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values compfile.writeto(compfile_name, overwrite=True) msg = "\nFits file with calculated pathloss values of each slit saved as: " print(msg) log_msgs.append(msg) print(outfile_name) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: " print(msg) log_msgs.append(msg) print(compfile_name) log_msgs.append(compfile_name) # If all tests passed then pytest will be marked as PASSED, else FAILED FINAL_TEST_RESULT = False for t in total_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break else: FINAL_TEST_RESULT = True if FINAL_TEST_RESULT: msg = "\n *** Final pathloss test result reported as PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slits PASSED path_loss test." else: msg = "\n *** Final pathloss test result reported as FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slits FAILED path_loss test." # end the timer pathloss_end_time = time.time() - pathtest_start_time if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60.0 # in minutes pathloss_tot_time = "* Script IFU_PS.py took ", repr( pathloss_end_time) + " minutes to finish." if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60. # in hours pathloss_tot_time = "* Script IFU_PS.py took ", repr( pathloss_end_time) + " hours to finish." else: pathloss_tot_time = "* Script IFU_PS.py took ", repr( pathloss_end_time) + " seconds to finish." print(pathloss_tot_time) log_msgs.append(pathloss_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs
hd = fits.open('data/' + disk_name[i] + '_resid.JvMcorr.fits')[0].header nx, ny = hd['NAXIS1'], hd['NAXIS2'] RAo = 3600 * hd['CDELT1'] * (np.arange(nx) - (hd['CRPIX1'] - 1)) DECo = 3600 * hd['CDELT2'] * (np.arange(ny) - (hd['CRPIX2'] - 1)) dRA, dDEC = np.meshgrid(RAo - offs[0], DECo - offs[1]) # beam parameters bmaj, bmin, bPA = 3600 * hd['BMAJ'], 3600 * hd['BMIN'], hd['BPA'] barea = (np.pi * bmaj * bmin / (4 * np.log(2))) / (3600 * 180 / np.pi)**2 # image setups im_bounds = (dRA.max(), dRA.min(), dDEC.min(), dDEC.max()) dRA_lims, dDEC_lims = [rs * rout, -rs * rout], [-rs * rout, rs * rout] # intensity limits, and stretch norm = ImageNormalize(vmin=-vspan, vmax=vspan, stretch=LinearStretch()) # set location in figure ax = fig.add_subplot(gs[np.floor_divide(i, 2), i % 2]) # load image hdu = fits.open('data/' + disk_name[i] + '_resid.JvMcorr.fits') img = 1e6 * np.squeeze(hdu[0].data) / 6. # plot the image (in uJy / beam units) im = ax.imshow(img, origin='lower', cmap=cmap, extent=im_bounds, norm=norm, aspect='equal')
xd = (dRA * np.cos(PAr) - dDEC * np.sin(PAr)) / np.cos(inclr) yd = (dRA * np.sin(PAr) + dDEC * np.cos(PAr)) r, theta = np.sqrt(xd**2 + yd**2), np.degrees(np.arctan2(yd, xd)) # beam parameters bmaj, bmin, bPA = 3600 * hd['BMAJ'], 3600 * hd['BMIN'], hd['BPA'] beam_area = (np.pi * bmaj * bmin / (4 * np.log(2))) / (3600 * 180 / np.pi)**2 # image setups rout = disk.disk[target]['rout'] im_bounds = (dRA.max(), dRA.min(), dDEC.min(), dDEC.max()) dRA_lims, dDEC_lims = [1.2 * rout, -1.2 * rout], [-1.2 * rout, 1.2 * rout] # intensity limits, and stretch norm = ImageNormalize(vmin=0, vmax=disk.disk[target]['maxTb'], stretch=AsinhStretch()) cmap = 'inferno' ### Plot the data image plt.style.use('default') fig = plt.figure(figsize=(7.0, 5.9)) gs = gridspec.GridSpec(1, 2, width_ratios=(1, 0.04)) # image (sky-plane) ax = fig.add_subplot(gs[0, 0]) Tb = (1e-23 * dimg / beam_area) * c_**2 / (2 * k_ * freq**2) im = ax.imshow(Tb, origin='lower', cmap=cmap, extent=im_bounds,
def plot_all_params(image: Union[str, fits.hdu.HDUList], cat: Union[str, Table, np.ndarray], show: bool = True, cutout: bool = False, ra_key: str = "ALPHA_SKY", dec_key: str = "DELTA_SKY", a_key: str = "A_WORLD", b_key: str = "B_WORLD", theta_key: str = "THETA_WORLD", kron: bool = False, kron_key: str = "KRON_RADIUS"): """ Plots :param image: :param cat: :param cutout: Plots a cutout of the image centre. Forces 'show' to True. :return: """ if cutout: show = True image, path = ff.path_or_hdu(image) data = image[0].data wcs_image = wcs.WCS(header=image[0].header) plt.subplot(projection=wcs_image) norm = ImageNormalize(data, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(data, origin='lower', norm=norm, ) plot_gal_params(hdu=image, ras=cat[ra_key], decs=cat[dec_key], a=cat[a_key], b=cat[b_key], theta=cat[theta_key], colour='red') if kron: plot_gal_params(hdu=image, ras=cat[ra_key], decs=cat[dec_key], a=cat[kron_key] * cat[a_key], b=cat[kron_key] * cat[b_key], theta=cat[theta_key], colour='violet') if show: plt.show() if cutout: n_x = data.shape[1] n_y = data.shape[0] mid_x = int(n_x / 2) mid_y = int(n_y / 2) left = mid_x - 45 right = mid_x + 45 bottom = mid_y - 45 top = mid_y + 45 gal = ff.trim(hdu=image, left=left, right=right, bottom=bottom, top=top) plt.imshow(gal[0].data) plot_gal_params(hdu=gal, ras=cat[ra_key], decs=cat[dec_key], a=cat[a_key], b=cat[b_key], theta=cat[theta_key], colour='red') if kron: plot_gal_params(hdu=image, ras=cat[ra_key], decs=cat[dec_key], a=cat[kron_key] * cat[a_key], b=cat[kron_key] * cat[b_key], theta=cat[theta_key], colour='violet') if show: plt.show() if path: image.close()
def pathtest(step_input_filename, reffile, comparison_filename, writefile=True, show_figs=False, save_figs=True, threshold_diff=1e-7, debug=False): """ This function calculates the difference between the pipeline and the calculated pathloss values. The functions use the output of the compute_world_coordinates.py script. Args: step_input_filename: str, name of the output fits file from the source type step (with full path) reffile: str, path to the pathloss FS reference fits file comparison_filename: str, path to comparison pipeline pathloss file writefile: boolean, if True writes the fits files of the calculated pathloss and difference image. show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots (the 3 plots can be saved or not independently with the function call) function will name the plot by default) threshold_diff: float, threshold difference between pipeline output and comparison file debug: boolean, if true a series of print statements will show on-screen Returns: - 1 plot, if told to save and/or show them. - median_diff: Boolean, True if smaller or equal to threshold. - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer pathtest_start_time = time.time() # get info from the rate file header det = fits.getval(step_input_filename, "DETECTOR", 0) msg = 'step_input_filename=' + step_input_filename print(msg) log_msgs.append(msg) exptype = fits.getval(step_input_filename, "EXP_TYPE", 0) grat = fits.getval(step_input_filename, "GRATING", 0) filt = fits.getval(step_input_filename, "FILTER", 0) msg = "path_loss_file --> Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype print(msg) log_msgs.append(msg) is_point_source = True # get the datamodel from the assign_wcs output file extract2d_wcs_file = step_input_filename.replace("srctype.fits", "extract_2d.fits") model = datamodels.MultiSlitModel(extract2d_wcs_file) if writefile: # create the fits list to hold the calculated pathloss values for # each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create the fits list to hold the image of pipeline-calculated # difference values hdu0 = fits.PrimaryHDU() compfile = fits.HDUList() compfile.append(hdu0) # list to determine if pytest is passed or not total_test_result = [] # loop over the slits sltname_list = ["S200A1", "S200A2", "S400A1", "S1600A1"] msg = "Now looping through the slits. This may take a while... " print(msg) log_msgs.append(msg) if det == "NRS2": sltname_list.append("S200B1") # but check if data is BOTS if fits.getval(step_input_filename, "EXP_TYPE", 0) == "NRS_BRIGHTOBJ": sltname_list = ["S1600A1"] # get all the science extensions ps_uni_ext_list = get_ps_uni_extensions(reffile, is_point_source) # get files print("""Checking if files exist & obtaining datamodels. This takes a few minutes...""") if os.path.isfile(comparison_filename): if debug: print('Comparison file does exist.') else: result_msg = 'Comparison file does NOT exist. Skipping pathloss test.' print(result_msg) log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the comparison data model pathloss_pipe = datamodels.open(comparison_filename) # For the moment, the pipeline is using the wrong reference file for slit 400A1, so read file that # re-processed with the right reference file and open corresponding data model # BUT we are skipping this since this is not in the released candidate of the pipeline """ pathloss_400a1 = step_input_filename.replace("srctype.fits", "pathloss_400A1.fits") pathloss_pipe_400a1 = datamodels.open(pathloss_400a1) """ if debug: print('got comparison datamodel!') if os.path.isfile(step_input_filename): if debug: print('Input file does exist.') else: result_msg = 'Input file does NOT exist. Skipping pathloss test.' log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the input data model pl = datamodels.open(step_input_filename) if debug: print('got input datamodel!') # loop through the wavelengths msg = "Looping through the wavelengths... " print(msg) log_msgs.append(msg) slit_val = 0 for slit, pipe_slit in zip(pl.slits, pathloss_pipe.slits): slit_val = slit_val + 1 slit_id = pipe_slit.name # with the current reference file, skip S400A1 #if slit_id == "S400A1": # continue print('\nWorking with slitlet ', slit_id) if slit.name == slit_id: msg = """Slitlet name in fits file previous to pathloss and in pathloss output file are the same. """ log_msgs.append(msg) print(msg) else: msg = """* Missmatch of slitlet names in fits file previous to pathloss and in pathloss output file. Skipping test. """ result = 'skip' log_msgs.append(msg) return result, msg, log_msgs # S-flat mode = "FS" if debug: print("grat = ", grat) continue_pl_test = False if fits.getval(step_input_filename, "EXP_TYPE", 0) == "NRS_BRIGHTOBJ": slit = model continue_pl_test = True else: for slit_in_MultiSlitModel in pl.slits: if slit_in_MultiSlitModel.name == slit_id: slit = slit_in_MultiSlitModel continue_pl_test = True break if not continue_pl_test: continue else: try: if is_point_source is True: ext = ps_uni_ext_list[0][slit_id] print("Retrieved point source extension") elif is_point_source is False: ext = ps_uni_ext_list[1][slit_id] print("WARNING: Retrieved extended source extension") except KeyError: ext = sltname_list.index(slit_id) print("Unable to retrieve extension.") wcs_obj = slit.meta.wcs # get the wavelength x, y = wcstools.grid_from_bounding_box(wcs_obj.bounding_box, step=(1, 1), center=True) ra, dec, wave = wcs_obj(x, y) # wave is in microns wave_sci = wave * 10**(-6) # microns --> meters # get positions of source in file: slit_x = slit.source_xpos slit_y = slit.source_ypos if debug: print("slit_x, slit_y", slit_x, slit_y) """ if slit_id == "S400A1": if is_point_source: ext = 1 else: ext = 3 if os.path.isfile("jwst-nirspec-a400.plrf.fits"): reffile2use = "jwst-nirspec-a400.plrf.fits" else: msg = "Skipping slit S400A1 because reference file not present" print(msg) log_msgs.append(msg) continue else:""" reffile2use = reffile msg = "Using reference file: " + reffile2use print(msg) log_msgs.append(msg) plcor_ref_ext = fits.getdata(reffile2use, ext) if debug: print("ext:", ext) hdul = fits.open(reffile2use) plcor_ref = hdul[1].data w = wcs.WCS(hdul[1].header) # make cube w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref. shape[1], :plcor_ref.shape[2]] slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0) previous_sci = slit.data pipe_correction = pipe_slit.pathloss """ if slit_id == "S400A1": for pipe_slit_400a1 in pathloss_pipe_400a1.slits: if pipe_slit_400a1.name == "S400A1": pipe_correction = pipe_slit_400a1.pathloss break else: continue """ if len(pipe_correction) == 0: print( "Pipeline pathloss correction in datamodel is empty. Skipping testing this slit." ) continue # Set up manually to test correction at nonzero point # slit_x = 0.2 # slit_y = 0.2 # if debug: # print("""WARNING: Using manually set slit_x and slit_y! # The pipeline correction will not use manually set values and # thus the residuals will change # """) correction_array = np.array([]) lambda_array = np.array([]) wave_sci_flat = wave_sci.reshape(wave_sci.size) wave_ref_flat = wave_ref.reshape(wave_ref.size) ref_xy = np.column_stack((slitx_ref.reshape(slitx_ref.size), slity_ref.reshape(slitx_ref.size))) # loop through slices in lambda from reference file shape = 0 for lambda_val in wave_ref_flat: # loop through every lambda value # flattened so that looping works smoothly shape = shape + 1 index = np.where(wave_ref[:, 0, 0] == lambda_val) # index of closest lambda value in reffile to given sci lambda # took index of only the first slice of wave_ref because # the others were repetitive & we got extra indices # take slice where lambda=index: plcor_slice = plcor_ref_ext[index[0][0]].reshape( plcor_ref_ext[index[0][0]].size) # do 2d interpolation to get a single correction factor for each slice corr_val = scipy.interpolate.griddata(ref_xy[:plcor_slice.size], plcor_slice, np.asarray([slit_x, slit_y]), method='linear') # append values from loop to create a vector of correction factors correction_array = np.append(correction_array, corr_val[0]) # map to array with corresponding lambda lambda_array = np.append(lambda_array, lambda_val) # get correction value for each pixel corr_vals = np.interp(wave_sci_flat, lambda_array, correction_array) corr_vals = corr_vals.reshape(wave_sci.shape) corrected_array = previous_sci / corr_vals # set up generals for all the plots font = {'weight': 'normal', 'size': 7} matplotlib.rc('font', **font) # Plots: step_input_filepath = step_input_filename.replace(".fits", "") # my correction values fig = plt.figure() plt.subplot(221) norm = ImageNormalize(corr_vals) plt.imshow(corr_vals, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Calculated Correction') plt.colorbar() # pipe correction plt.subplot(222) norm = ImageNormalize(pipe_correction) plt.imshow(pipe_correction, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.title("Pipeline Correction") plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.colorbar() # residuals (pipe correction - my correction) corr_residuals = pipe_correction - corr_vals plt.subplot(223) norm = ImageNormalize(corr_residuals) plt.imshow(corr_residuals, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.title('Correction residuals') plt.colorbar() # my science data after plt.subplot(224) norm = ImageNormalize(corrected_array) plt.imshow(corrected_array, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.title('My science data after pathloss') plt.xlabel('dispersion in pixels') plt.ylabel('y in pixels') plt.colorbar() fig.suptitle("FS PS Pathloss Correction Test for slit " + str(slit_id)) if save_figs: plt_name = step_input_filepath + "_Pathloss_test_slitlet_"+str(mode) + "_" + str(slit_id) + "_" + \ str(slit_val) + ".png" plt.savefig(plt_name) print('Figure saved as: ', plt_name) if show_figs: plt.show() elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." if debug: print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." if debug: print(msg) log_msgs.append(msg) plt.clf() # create fits file to hold the calculated pathloss for each slit if writefile: msg = "Saving the fits files with the calculated pathloss for each slit..." print(msg) log_msgs.append(msg) # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(corr_vals, name=slit_id) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values compfile_ext = fits.ImageHDU(corr_residuals, name=slit_id) compfile.append(compfile_ext) if corr_residuals[~np.isnan(corr_residuals)].size == 0: msg1 = " * Unable to calculate statistics because difference array has all values as NaN. " \ "Test will be set to FAILED." print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: msg = "Calculating statistics... " print(msg) log_msgs.append(msg) corr_residuals = corr_residuals[ np.where((corr_residuals != 999.0) & (corr_residuals < 0.1) & (corr_residuals > -0.1))] # ignore outliers if corr_residuals.size == 0: msg1 = " * Unable to calculate statistics because difference array has all outlier values. " \ "Test will be set to FAILED." print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: stats_and_strings = auxfunc.print_stats(corr_residuals, "Difference", float(threshold_diff), absolute=True) stats, stats_print_strings = stats_and_strings corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats for msg in stats_print_strings: log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(corr_residuals_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) total_test_result.append(test_result) if writefile: outfile_name = step_input_filename.replace("srctype", "calcuated_pathloss") compfile_name = step_input_filename.replace("srctype", "comparison_pathloss") # create the fits list to hold the calculated pathloss values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values compfile.writeto(compfile_name, overwrite=True) msg = "\nFits file with calculated pathloss values of each slit saved as: " print(msg) log_msgs.append(msg) print(outfile_name) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: " print(msg) log_msgs.append(msg) print(compfile_name) log_msgs.append(compfile_name) # If all tests passed then pytest is marked as PASSED, else it is FAILED FINAL_TEST_RESULT = False for t in total_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break else: FINAL_TEST_RESULT = True if FINAL_TEST_RESULT: msg = "\n *** Final pathloss test result reported as PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slits PASSED path_loss test." else: msg = "\n *** Final pathloss test result reported as FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slits FAILED path_loss test." # end the timer pathloss_end_time = time.time() - pathtest_start_time if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60.0 # in minutes pathloss_tot_time = "* Script FS_PS.py took ", repr( pathloss_end_time) + " minutes to finish." if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60. # in hours pathloss_tot_time = "* Script FS_PS.py took ", repr( pathloss_end_time) + " hours to finish." else: pathloss_tot_time = "* Script FS_PS.py took ", repr( pathloss_end_time) + " seconds to finish." print(pathloss_tot_time) log_msgs.append(pathloss_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs
def plot_hst_loc_cartopy(self, i=5, df=None, title='', thresh=5, fout='', min_exptime=800, key='start', save=False, orbital_path1=None, orbital_path2=None, projection=ccrs.PlateCarree()): fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(8, 7), tight_layout=True, subplot_kw={'projection': projection}) crs = projection transform = crs._as_mpl_transform(ax) df = df[df.integration_time.gt(min_exptime)] df.sort_values(by='incident_cr_rate', inplace=True) # Plot configuration ax.coastlines() gl = ax.gridlines(crs=crs, draw_labels=True, linewidth=1, color='k', alpha=0.4, linestyle='--') fname = '/ifs/missions/projects/plcosmic/hst_cosmic_rays/APJ_plots/HYP_50M_SR_W.tif' ax.imshow(plt.imread(fname), origin='upper', transform=crs, extent=[-180, 180, -90, 90]) gl.xlabels_top = False gl.ylabels_left = True gl.ylabels_right = False gl.xlines = True # gl.xlocator = mticker.FixedLocator([-180, -45, 0, 45, 180]) gl.xformatter = LONGITUDE_FORMATTER gl.yformatter = LATITUDE_FORMATTER gl.xlocator = MultipleLocator(60) gl.ylocator = MultipleLocator(15) gl.xlabel_style = {'size': 10, 'color': 'black'} gl.xlabel_style = {'color': 'black'} date = 2005 altitude = 565 # Calculate the B field grid # Evenly space grid with 1 degree resolution in both Latitude and Longitude lat = np.linspace(-90, 90, 1 * 180 + 1) lon = np.linspace(0, 360, 1 * 360 + 1) lat_grid, lon_grid = np.meshgrid(lat, lon) coordinates = list(zip(lat_grid.ravel(), lon_grid.ravel())) B_strength = [] for coords in coordinates: b_field = ipmag.igrf([date, altitude, coords[0], coords[1]]) B_strength.append(b_field[-1]) B_strength_grid = np.array(B_strength).reshape(lat_grid.shape) # Get the CR rate information lat, lon, rate = df['latitude_{}'.format(key)], \ df['longitude_{}'.format(key)], \ df['incident_cr_rate'] LOG.info('{} {} {}'.format(len(lat), len(lon), len(rate))) # Get average statistics to generate contour mean, median, std = sigma_clipped_stats(rate, sigma_lower=3, sigma_upper=3) LOG.info('{} +\- {}'.format(mean, std)) norm = ImageNormalize(rate, stretch=LinearStretch(), vmin=mean - thresh * std, vmax=mean + thresh * std) cbar_below_mean = [mean - (i + 1) * std for i in range(thresh)] cbar_above_mean = [mean + (i + 1) * std for i in range(thresh)] cbar_bounds = cbar_below_mean + [mean] + cbar_above_mean print(cbar_bounds) cbar_bounds.sort() sci_cmap = plt.cm.viridis custom_norm = colors.BoundaryNorm(boundaries=cbar_bounds, ncolors=sci_cmap.N) scat = ax.scatter(lon.values, lat.values, marker='o', s=3.5, c=rate, alpha=0.2, norm=custom_norm, cmap='viridis', transform=ccrs.PlateCarree()) cbar_ticks = cbar_bounds cax = fig.add_axes([0.1, 0.2, 0.8, 0.05]) cbar = fig.colorbar(scat, cax=cax, ticks=cbar_ticks, orientation='horizontal') cbar.set_alpha(1) cbar.draw_all() cbar_tick_labels = [f'<x>-{i}$\sigma$' for i in [5, 4, 3, 2, 1]] + [ '<x>' ] + [f'<x>+{i}$\sigma$' for i in [1, 2, 3, 4, 5]] cbar.ax.set_xticklabels(cbar_tick_labels, horizontalalignment='right', rotation=30) cbar.set_label('CR Flux [CR/s/$cm^2$]', fontsize=10) cntr = ax.contour(lon_grid, lat_grid, B_strength_grid, cmap='plasma', levels=10, alpha=1, lw=2, transform=ccrs.PlateCarree()) h1, l1 = cntr.legend_elements("B_strength_grid") l1_custom = [ f"{val.split('=')[-1].strip('$').strip()} nT" for val in l1 ] leg1 = Legend(ax, h1, l1_custom, loc='upper left', edgecolor='k', fontsize=8, framealpha=0.45, facecolor='tab:gray', bbox_to_anchor=(1.05, 1.03), title='Total Magnetic Intensity') ax.add_artist(leg1) if orbital_path1 is not None: ax.scatter(orbital_path1.metadata['longitude'][::4][1:], orbital_path1.metadata['latitude'][::4][1:], c='k', s=20, label='285 seccond interval') if orbital_path2 is not None: ax.plot(orbital_path2.metadata['longitude'], orbital_path2.metadata['latitude'], label=f'Orbital Path Over {2000:.0f} seconds', color='k', ls='--', lw=1.25) plt.show() return fig
a.jsoc.Notify(jsoc_email), a.jsoc.Segment.image, cutout, ) ##################################################### # Submit the export request and download the data. files = Fido.fetch(query) files.sort() ##################################################### # Now that we've downloaded the files, we can create # a `~sunpy.map.MapSequence` from them. sequence = sunpy.map.Map(files, sequence=True) ##################################################### # Finally, we can construct an animation in time from # our stack of cutouts and interactively flip through # each image in our sequence. We first adjust the plot # settings on each image to ensure the colorbar is the # same at each time step. for each_map in sequence: each_map.plot_settings['norm'] = ImageNormalize(vmin=0, vmax=5e3, stretch=SqrtStretch()) plt.figure() ani = sequence.plot() plt.show()
def defringeflat(flat_file, wbin=10, start_col=10, end_col=980, clip1=0, diagnostic=True, save_to_path=None, filename=None): """ This function is to remove the fringe pattern using the method described in Rojo and Harrington (2006). Use a fifth order polynomial to remove the continuum. Parameters ---------- flat_file : fits original flat file Optional Parameters ------------------- wbin : int the bin width to calculate each enhance row Default is 32 start_col : int starting column number for the wavelet analysis Default is 10 end_col : int ending column number for the wavelet analysis Default is 980 diagnostic : boolean output the diagnostic plots Default is True Returns ------- defringe file : fits defringed flat file """ # the path to save the diagnostic plots #save_to_path = 'defringeflat/allflat/' #print(flat_file) data = fits.open(flat_file, ignore_missing_end=True) # Use the data to figure out the values to mask through the image (low counts/order edges) hist, bins = np.histogram(data[0].data.flatten(), bins=int(np.sqrt(len(data[0].data.flatten())))) bins = bins[0:-1] index1 = np.where((bins > np.percentile(data[0].data.flatten(), 10)) & (bins < np.percentile(data[0].data.flatten(), 30))) try: lowval = bins[index1][np.where(hist[index1] == np.min(hist[index1]))] #print(lowval, len(lowval)) if len(lowval) >= 2: lowval = np.min(lowval) except: lowval = 0 #if no values for index1 flat = data # initial flat plot if diagnostic is True: # Save the images to a separate folder save_to_image_path = save_to_path + '/images/' if not os.path.exists(save_to_image_path): os.makedirs(save_to_image_path) fig = plt.figure(figsize=(8, 8)) fig.suptitle("original flat", fontsize=12) gs = gridspec.GridSpec(2, 1, height_ratios=[6, 1]) ax0 = plt.subplot(gs[0]) # Create an ImageNormalize object norm = ImageNormalize(flat[0].data, interval=ZScaleInterval()) ax0.imshow(flat[0].data, cmap='gray', norm=norm, origin='lower', aspect='auto') ax0.set_ylabel("Row number") ax1 = plt.subplot(gs[1], sharex=ax0) ax1.plot(flat[0].data[60, :], 'k-', alpha=0.5, label='60th row profile') ax1.set_ylabel("Amp (DN)") ax1.set_xlabel("Column number") plt.legend() plt.savefig(save_to_image_path + "defringeflat_{}_0_original_flat.png"\ .format(filename), bbox_inches='tight') plt.close() defringeflat_img = data defringe_data = np.array(defringeflat_img[0].data, dtype=float) for k in np.arange(0, 1024 - wbin, wbin): #print(k) #if k != 310: continue """ # Use the data to figure out the values to mask through the image (low counts/order edges) hist, bins = np.histogram(flat[0].data[k:k+wbin+1, 0:1024-clip1].flatten(), bins=int(np.sqrt(len(flat[0].data[k:k+wbin+1, 0:1024-clip1].flatten())))) bins = bins[0:-1] index1 = np.where( (bins > np.percentile(flat[0].data[k:k+wbin+1, 0:1024-clip1].flatten(), 10)) & (bins < np.percentile(flat[0].data[k:k+wbin+1, 0:1024-clip1].flatten(), 30)) ) lowval = bins[index1][np.where(hist[index1] == np.min(hist[index1]))] #print(lowval, len(lowval)) if len(lowval) >= 2: lowval = np.min(lowval) """ # Find the mask mask = np.zeros(flat[0].data[k:k + wbin + 1, 0:1024 - clip1].shape) baddata = np.where( flat[0].data[k:k + wbin + 1, 0:1024 - clip1] <= lowval) mask[baddata] = 1 # extract the patch from the fits file #flat_patch = np.ma.array(flat[0].data[k:k+wbin,:], mask=mask) flat_patch = np.array(flat[0].data[k:k + wbin + 1, 0:1024 - clip1]) # median average the selected region in the order flat_patch_median = np.ma.median(flat_patch, axis=0) # continuum fit # smooth the continuum (Chris's method) smoothed = sp.ndimage.uniform_filter1d(flat_patch_median, 30) splinefit = sp.interpolate.interp1d(np.arange(len(smoothed)), smoothed, kind='cubic') cont_fit = splinefit(np.arange(0, 1024 - clip1)) #smoothed # Now fit a polynomial #pcont = np.ma.polyfit(np.arange(0, 1024-clip1), # cont_fit, 10) #cont_fit2 = np.polyval(pcont, np.arange(0,1024)) #plt.plot(flat_patch_median, c='r') #plt.plot(smoothed, c='b') #plt.savefig(save_to_image_path + "TEST.png", bbox_inches='tight') #plt.close() #plt.show() #sys.exit() #pcont = np.ma.polyfit(np.arange(start_col,end_col), # flat_patch_median[start_col:end_col],10) #cont_fit = np.polyval(pcont, np.arange(0,1024)) # use wavelets package: WaveletAnalysis enhance_row = flat_patch_median - cont_fit dt = 0.1 wa = WaveletAnalysis(enhance_row[start_col:end_col], dt=dt) # wavelet power spectrum power = wa.wavelet_power # scales cales = wa.scales # associated time vector t = wa.time # reconstruction of the original data rx = wa.reconstruction() # reconstruct the fringe image reconstruct_image = np.zeros(defringe_data[k:k + wbin + 1, 0:1024 - clip1].shape) for i in range(wbin + 1): for j in np.arange(start_col, end_col): reconstruct_image[i, j] = rx[j - start_col] defringe_data[k:k + wbin + 1, 0:1024 - clip1] -= reconstruct_image[0:1024 - clip1] # Add in something for the edges/masked out data in the reconstructed image defringe_data[k:k + wbin + 1, 0:1024 - clip1][baddata] = flat[0].data[k:k + wbin + 1, 0:1024 - clip1][baddata] #print("{} row starting {} is done".format(filename,k)) # diagnostic plots if diagnostic is True: print("Generating diagnostic plots") # middle cut plot fig = plt.figure(figsize=(10, 6)) fig.suptitle("middle cut at row {}".format(k + wbin // 2), fontsize=12) ax1 = fig.add_subplot(2, 1, 1) norm = ImageNormalize(flat_patch, interval=ZScaleInterval()) ax1.imshow(flat_patch, cmap='gray', norm=norm, origin='lower', aspect='auto') ax1.set_ylabel("Row number") ax2 = fig.add_subplot(2, 1, 2, sharex=ax1) ax2.plot(flat_patch[wbin // 2, :], 'k-', alpha=0.5) ax2.set_ylabel("Amp (DN)") ax2.set_xlabel("Column number") plt.tight_layout() plt.subplots_adjust(top=0.85, hspace=0.5) plt.savefig(save_to_image_path + \ 'defringeflat_{}_flat_start_row_{}_middle_profile.png'\ .format(filename,k), bbox_inches='tight') plt.close() # continuum fit plot fig = plt.figure(figsize=(10, 6)) fig.suptitle("continuum fit row {}-{}".format(k, k + wbin), fontsize=12) gs = gridspec.GridSpec(2, 1, height_ratios=[3, 1]) ax0 = plt.subplot(gs[0]) ax0.plot(flat_patch_median, 'k-', alpha=0.5, label='mean average patch') ax0.plot(cont_fit, 'r-', alpha=0.5, label='continuum fit') #ax0.plot(cont_fit2,'m-', alpha=0.5, label='continuum fit poly') ax0.set_ylabel("Amp (DN)") plt.legend() ax1 = plt.subplot(gs[1]) ax1.plot(flat_patch_median - cont_fit, 'k-', alpha=0.5, label='residual') ax1.set_ylabel("Amp (DN)") ax1.set_xlabel("Column number") plt.legend() plt.tight_layout() plt.subplots_adjust(top=0.85, hspace=0.5) plt.savefig(save_to_image_path + \ "defringeflat_{}_start_row_{}_continuum_fit.png".\ format(filename,k), bbox_inches='tight') #plt.show() #sys.exit() plt.close() # enhance row vs. reconstructed wavelet plot try: fig = plt.figure(figsize=(10, 6)) fig.suptitle("reconstruct fringe comparison row {}-{}".\ format(k,k+wbin), fontsize=10) ax1 = fig.add_subplot(3, 1, 1) ax1.set_title('enhance_row start row') ax1.plot(enhance_row, 'k-', alpha=0.5, label="enhance_row start row {}".format(k)) ax1.set_ylabel("Amp (DN)") #plt.legend() ax2 = fig.add_subplot(3, 1, 2, sharex=ax1) ax2.set_title('reconstructed fringe pattern') ax2.plot(rx, 'k-', alpha=0.5, label='reconstructed fringe pattern') ax2.set_ylabel("Amp (DN)") #plt.legend() ax3 = fig.add_subplot(3, 1, 3, sharex=ax1) ax3.set_title('residual') ax3.plot(enhance_row[start_col:end_col] - rx, 'k-', alpha=0.5, label='residual') ax3.set_ylabel("Amp (DN)") ax3.set_xlabel("Column number") #plt.legend() plt.tight_layout() plt.subplots_adjust(top=0.85, hspace=0.5) plt.savefig(save_to_image_path + \ "defringeflat_{}_start_row_{}_reconstruct_profile.png".\ format(filename,k), bbox_inches='tight') plt.close() except RuntimeError: print("CANNOT GENERATE THE PLOT defringeflat\ _{}_start_row_{}_reconstruct_profile.png" \ .format(filename,k)) pass # reconstruct image comparison plot fig = plt.figure(figsize=(10, 6)) fig.suptitle("reconstructed image row {}-{}".\ format(k,k+wbin), fontsize=12) ax1 = fig.add_subplot(3, 1, 1) ax1.set_title('raw flat image') norm = ImageNormalize(flat_patch, interval=ZScaleInterval()) ax1.imshow(flat_patch, cmap='gray', norm=norm, origin='lower', label='raw flat image', aspect='auto') ax1.set_ylabel("Row number") #plt.legend() ax2 = fig.add_subplot(3, 1, 2, sharex=ax1) ax2.set_title('reconstructed fringe image') norm = ImageNormalize(reconstruct_image, interval=ZScaleInterval()) ax2.imshow(reconstruct_image, cmap='gray', norm=norm, origin='lower', label='reconstructed fringe image', aspect='auto') ax2.set_ylabel("Row number") #plt.legend() ax3 = fig.add_subplot(3, 1, 3, sharex=ax1) ax3.set_title('residual') norm = ImageNormalize(flat_patch - reconstruct_image, interval=ZScaleInterval()) ax3.imshow(flat_patch - reconstruct_image, norm=norm, origin='lower', cmap='gray', label='residual', aspect='auto') ax3.set_ylabel("Row number") ax3.set_xlabel("Column number") #plt.legend() plt.tight_layout() plt.subplots_adjust(top=0.85, hspace=0.5) plt.savefig(save_to_image_path + \ "defringeflat_{}_start_row_{}_reconstruct_image.png".\ format(filename,k), bbox_inches='tight') plt.close() # middle residual comparison plot fig = plt.figure(figsize=(10, 6)) fig.suptitle("middle row comparison row {}-{}".\ format(k,k+wbin), fontsize=12) ax1 = fig.add_subplot(3, 1, 1) ax1.plot(flat_patch[wbin // 2, :], 'k-', alpha=0.5, label='original flat row {}'.format(k + wbin / 2)) ax1.set_ylabel("Amp (DN)") plt.legend() ax2 = fig.add_subplot(3, 1, 2, sharex=ax1) ax2.plot(flat_patch[wbin//2,:]-\ reconstruct_image[wbin//2,:],'k-', alpha=0.5, label='defringed flat row {}'.format(k+wbin/2)) ax2.set_ylabel("Amp (DN)") plt.legend() ax3 = fig.add_subplot(3, 1, 3, sharex=ax1) ax3.plot(reconstruct_image[wbin // 2, :], 'k-', alpha=0.5, label='difference') ax3.set_ylabel("Amp (DN)") ax3.set_xlabel("Column number") plt.legend() plt.tight_layout() plt.subplots_adjust(top=0.85, hspace=0.5) plt.savefig(save_to_image_path + \ "defringeflat_{}_start_row_{}_defringe_middle_profile.png".\ format(filename,k), bbox_inches='tight') plt.close() #if k > 30: sys.exit() # for testing purposes # final diagnostic plot if diagnostic is True: fig = plt.figure(figsize=(8, 8)) fig.suptitle("defringed flat", fontsize=12) gs = gridspec.GridSpec(2, 1, height_ratios=[6, 1]) ax0 = plt.subplot(gs[0]) norm = ImageNormalize(defringe_data, interval=ZScaleInterval()) ax0.imshow(defringe_data, cmap='gray', norm=norm, origin='lower', aspect='auto') ax0.set_ylabel("Row number") ax1 = plt.subplot(gs[1], sharex=ax0) ax1.plot(defringe_data[60, :], 'k-', alpha=0.5, label='60th row profile') ax1.set_ylabel("Amp (DN)") ax1.set_xlabel("Column number") plt.legend() plt.tight_layout() plt.subplots_adjust(top=0.85, hspace=0.5) plt.savefig(save_to_image_path + "defringeflat_{}_0_defringe_flat.png"\ .format(filename), bbox_inches='tight') plt.close() hdu = fits.PrimaryHDU(data=defringe_data) hdu.header = flat[0].header return hdu
def add_FlatSummary(filenames, MasterFlat, html_fileimagestretch='linear'): """ add bias processing summary to website """ if not os.path.isdir('diagnostics'): os.mkdir('diagnostics') hdulist = fits.open(MasterFlat) framefilename = 'diagnostics/' + MasterFlat + '.png' imgdat = hdulist[0].data html = '<H2>Flat Processing summary</H2>' html += '<p> %d flats have been processed </p>' % \ (len(filenames)) html += '<p> Master flat file created as <A HREF=\"%s\">%s</A> </p>' % \ (framefilename, MasterFlat) html += ('<p><IMG SRC=\"%s\"></p>')% \ (framefilename) html += '\n' html += 'Statistics of the master flat:' html += "<P><TABLE BORDER=\"1\">\n<TR>\n" html += ("<TR><TD>Mean</TD><TD>%f</TD></TR><TR><TD>Median</TD><TD>%f</TD></TR><TR><TD>Std</TD><TD>%f</TD></TR>\n") % \ (np.nanmean(imgdat),np.nanmedian(imgdat),np.nanstd(imgdat)) html += '</TABLE>\n' ### Create frame image imgdat = hdulist[0].data # clip extreme values to prevent crash of imresize imgdat = np.clip(imgdat, np.percentile(imgdat, 1), np.percentile(imgdat, 99)) imgdat = imresize(imgdat, min(1., 1000. / np.max(imgdat.shape)), interp='nearest') #resize image larger than 1000px on one side norm = ImageNormalize(imgdat, interval=ZScaleInterval(), stretch={ 'linear': LinearStretch(), 'log': LogStretch() }[imagestretch]) plt.figure(figsize=(5, 5)) img = plt.imshow(imgdat, cmap='gray', norm=norm, origin='lower') # remove axes plt.axis('off') img.axes.get_xaxis().set_visible(False) img.axes.get_yaxis().set_visible(False) plt.savefig(framefilename, format='png', bbox_inches='tight', pad_inches=0, dpi=200) plt.close() hdulist.close() del (imgdat) append_website(html_file, html, replace_below="<H2>Flat Processing summary</H2>\n") return None
# %% psf = aiapy.psf.psf(m.wavelength) # %% [markdown] # We'll plot just a 500-by-500 pixel section centered on the center pixel. The # diffraction "arms" extending from the center pixel can often be seen in # flare observations due to the intense, small-scale brightening. # # # %% fov = 500 lc_x, lc_y = psf.shape[0] // 2 - fov // 2, psf.shape[1] // 2 - fov // 2 plt.imshow(psf[lc_x:lc_x + fov, lc_y:lc_y + fov], norm=ImageNormalize(vmin=1e-8, vmax=1e-3, stretch=LogStretch())) plt.colorbar() plt.show() # %% [markdown] # Now that we've downloaded our image and computed the PSF, we can deconvolve # the image with the PSF using the # `Richardson-Lucy deconvolution algorithm <https://en.wikipedia.org/wiki/Richardson%E2%80%93Lucy_deconvolution>`_. # Note that passing in the PSF is optional. If you exclude it, it will be # calculated automatically. However, when deconvolving many images of the same # wavelength, it is most efficient to only calculate the PSF once. # # As with `~aiapy.psf.psf`, this will be much faster if you have # a GPU and `cupy` installed. # #
if args.wcs == 'True' or args.wcs == 'yes': fig, ax = plt.subplots(figsize=(7, 7)) ax = plt.subplot(projection=wcs_object) gaia = Irsa.query_region(SkyCoord(hdr[0].header['CRVAL1'] * u.deg, hdr[0].header['CRVAL2'] * u.deg, frame='fk5'), catalog="gaia_dr2_source", spatial="Cone", radius=4 * u.arcmin) if len(gaia) == 0: log.info('No GAIA stars found within 4 arcmin for starlist.') plt.close() else: ax.imshow(sci_med, cmap='gray', norm=ImageNormalize(sci_med, interval=ZScaleInterval())) _, median, std = sigma_clipped_stats(sci_med, sigma=3.0) daofind = DAOStarFinder(fwhm=7.0, threshold=5. * std) sources = daofind(np.asarray(sci_med)) for l, m in enumerate(gaia['source_id']): x, y = (wcs.WCS(hdr[0].header)).all_world2pix( gaia['ra'][l], gaia['dec'][l], 1) ax.add_patch( patches.Circle((x, y), radius=4, edgecolor='g', alpha=0.5, facecolor='none', linewidth=2, label='Gaia star: RA = %f, Dec = %f' % (gaia['ra'][l], gaia['dec'][l]),
def stack(files: list, output: str = None, directory: str = '', stack_type: str = 'median', inherit: bool = True, show: bool = False, normalise: bool = False): accepted_stack_types = ['mean', 'median', 'add'] if stack_type not in accepted_stack_types: raise ValueError('stack_type must be in ' + str(accepted_stack_types)) if directory != '' and directory[-1] != '/': directory = directory + '/' data = [] template = None print('Stacking:') for f in files: # Extract image data and append to list. if type(f) is str: f = u.sanitise_file_ext(f, 'fits') print(' ' + f) f = fits.open(directory + f) if type(f) is fits.hdu.hdulist.HDUList: data_append = f[0].data if template is None and inherit: # Get a template file to use for output. # TODO: Refine this - keep a record of which files went in in the header. template = f.copy() elif type(f) is CCDData: data_append = f.data else: raise TypeError( 'files must contain only strings, HDUList or CCDData objects.') if normalise: data_append = data_append / np.nanmedian( data_append[np.isfinite(data_append)]) if show: norm = ImageNormalize(data_append, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(data_append, origin='lower', norm=norm) plt.show() data.append(data_append) data = np.array(data) if stack_type == 'mean': stacked = np.mean(data, axis=0) elif stack_type == 'median': stacked = np.median(data, axis=0) else: stacked = np.sum(data, axis=0) if show: norm = ImageNormalize(stacked, interval=ZScaleInterval(), stretch=SqrtStretch()) plt.imshow(stacked, origin='lower', norm=norm) plt.show() if inherit: template[0].data = stacked else: template = fits.PrimaryHDU(stacked) template = fits.HDUList([template]) add_log(template, f'Stacked.') if output is not None: print('Writing stacked image to', output) template[0].header['BZERO'] = 0 template.writeto(output, overwrite=True, output_verify='warn') return template
def fits_finder_chart( fitsfile, outfile, fitsext=0, wcsfrom=None, scale=ZScaleInterval(), stretch=LinearStretch(), colormap=plt.cm.gray_r, findersize=None, finder_coordlimits=None, overlay_ra=None, overlay_decl=None, overlay_pltopts={'marker':'o', 'markersize':10.0, 'markerfacecolor':'none', 'markeredgewidth':2.0, 'markeredgecolor':'red'}, overlay_zoomcontain=False, grid=False, gridcolor='k' ): '''This makes a finder chart for a given FITS with an optional object position overlay. Parameters ---------- fitsfile : str `fitsfile` is the FITS file to use to make the finder chart. outfile : str `outfile` is the name of the output file. This can be a png or pdf or whatever else matplotlib can write given a filename and extension. fitsext : int Sets the FITS extension in `fitsfile` to use to extract the image array from. wcsfrom : str or None If `wcsfrom` is None, the WCS to transform the RA/Dec to pixel x/y will be taken from the FITS header of `fitsfile`. If this is not None, it must be a FITS or similar file that contains a WCS header in its first extension. scale : astropy.visualization.Interval object `scale` sets the normalization for the FITS pixel values. This is an astropy.visualization Interval object. See http://docs.astropy.org/en/stable/visualization/normalization.html for details on `scale` and `stretch` objects. stretch : astropy.visualization.Stretch object `stretch` sets the stretch function for mapping FITS pixel values to output pixel values. This is an astropy.visualization Stretch object. See http://docs.astropy.org/en/stable/visualization/normalization.html for details on `scale` and `stretch` objects. colormap : matplotlib Colormap object `colormap` is a matplotlib color map object to use for the output image. findersize : None or tuple of two ints If `findersize` is None, the output image size will be set by the NAXIS1 and NAXIS2 keywords in the input `fitsfile` FITS header. Otherwise, `findersize` must be a tuple with the intended x and y size of the image in inches (all output images will use a DPI = 100). finder_coordlimits : list of four floats or None If not None, `finder_coordlimits` sets x and y limits for the plot, effectively zooming it in if these are smaller than the dimensions of the FITS image. This should be a list of the form: [minra, maxra, mindecl, maxdecl] all in decimal degrees. overlay_ra, overlay_decl : np.array or None `overlay_ra` and `overlay_decl` are ndarrays containing the RA and Dec values to overplot on the image as an overlay. If these are both None, then no overlay will be plotted. overlay_pltopts : dict `overlay_pltopts` controls how the overlay points will be plotted. This a dict with standard matplotlib marker, etc. kwargs as key-val pairs, e.g. 'markersize', 'markerfacecolor', etc. The default options make red outline circles at the location of each object in the overlay. overlay_zoomcontain : bool `overlay_zoomcontain` controls if the finder chart will be zoomed to just contain the overlayed points. Everything outside the footprint of these points will be discarded. grid : bool `grid` sets if a grid will be made on the output image. gridcolor : str `gridcolor` sets the color of the grid lines. This is a usual matplotib color spec string. Returns ------- str or None The filename of the generated output image if successful. None otherwise. ''' # read in the FITS file if wcsfrom is None: hdulist = pyfits.open(fitsfile) img, hdr = hdulist[fitsext].data, hdulist[fitsext].header hdulist.close() frameshape = (hdr['NAXIS1'], hdr['NAXIS2']) w = WCS(hdr) elif os.path.exists(wcsfrom): hdulist = pyfits.open(fitsfile) img, hdr = hdulist[fitsext].data, hdulist[fitsext].header hdulist.close() frameshape = (hdr['NAXIS1'], hdr['NAXIS2']) w = WCS(wcsfrom) else: LOGERROR('could not determine WCS info for input FITS: %s' % fitsfile) return None # use the frame shape to set the output PNG's dimensions if findersize is None: fig = plt.figure(figsize=(frameshape[0]/100.0, frameshape[1]/100.0)) else: fig = plt.figure(figsize=findersize) # set the coord limits if zoomcontain is True # we'll leave 30 arcseconds of padding on each side if (overlay_zoomcontain and overlay_ra is not None and overlay_decl is not None): finder_coordlimits = [overlay_ra.min()-30.0/3600.0, overlay_ra.max()+30.0/3600.0, overlay_decl.min()-30.0/3600.0, overlay_decl.max()+30.0/3600.0] # set the coordinate limits if provided if finder_coordlimits and isinstance(finder_coordlimits, (list,tuple)): minra, maxra, mindecl, maxdecl = finder_coordlimits cntra, cntdecl = (minra + maxra)/2.0, (mindecl + maxdecl)/2.0 pixelcoords = w.all_world2pix([[minra, mindecl], [maxra, maxdecl], [cntra, cntdecl]],1) x1, y1, x2, y2 = (int(pixelcoords[0,0]), int(pixelcoords[0,1]), int(pixelcoords[1,0]), int(pixelcoords[1,1])) xmin = x1 if x1 < x2 else x2 xmax = x2 if x2 > x1 else x1 ymin = y1 if y1 < y2 else y2 ymax = y2 if y2 > y1 else y1 # create a new WCS with the same transform but new center coordinates whdr = w.to_header() whdr['CRPIX1'] = (xmax - xmin)/2 whdr['CRPIX2'] = (ymax - ymin)/2 whdr['CRVAL1'] = cntra whdr['CRVAL2'] = cntdecl whdr['NAXIS1'] = xmax - xmin whdr['NAXIS2'] = ymax - ymin w = WCS(whdr) else: xmin, xmax, ymin, ymax = 0, hdr['NAXIS2'], 0, hdr['NAXIS1'] # add the axes with the WCS projection # this should automatically handle subimages because we fix the WCS # appropriately above for these fig.add_subplot(111,projection=w) if scale is not None and stretch is not None: norm = ImageNormalize(img, interval=scale, stretch=stretch) plt.imshow(img[ymin:ymax,xmin:xmax], origin='lower', cmap=colormap, norm=norm) else: plt.imshow(img[ymin:ymax,xmin:xmax], origin='lower', cmap=colormap) # handle additional options if grid: plt.grid(color=gridcolor,ls='solid',lw=1.0) # handle the object overlay if overlay_ra is not None and overlay_decl is not None: our_pltopts = dict( transform=plt.gca().get_transform('fk5'), marker='o', markersize=10.0, markerfacecolor='none', markeredgewidth=2.0, markeredgecolor='red', rasterized=True, linestyle='none' ) if overlay_pltopts is not None and isinstance(overlay_pltopts, dict): our_pltopts.update(overlay_pltopts) plt.gca().set_autoscale_on(False) plt.gca().plot(overlay_ra, overlay_decl, **our_pltopts) plt.xlabel('Right Ascension [deg]') plt.ylabel('Declination [deg]') # get the x and y axes objects to fix the ticks xax = plt.gca().coords[0] yax = plt.gca().coords[1] yax.set_major_formatter('d.ddd') xax.set_major_formatter('d.ddd') # save the figure plt.savefig(outfile, dpi=100.0) plt.close('all') return outfile
async def get(self, object_id: str = None): """ --- summary: Serve alert cutout as fits or png tags: - alerts - kowalski parameters: - in: query name: instrument required: false schema: type: str - in: query name: candid description: "ZTF alert candid" required: true schema: type: integer - in: query name: cutout description: "retrieve science, template, or difference cutout image?" required: true schema: type: string enum: [science, template, difference] - in: query name: file_format description: "response file format: original loss-less FITS or rendered png" required: true default: png schema: type: string enum: [fits, png] - in: query name: interval description: "Interval to use when rendering png" required: false schema: type: string enum: [min_max, zscale] - in: query name: stretch description: "Stretch to use when rendering png" required: false schema: type: string enum: [linear, log, asinh, sqrt] - in: query name: cmap description: "Color map to use when rendering png" required: false schema: type: string enum: [bone, gray, cividis, viridis, magma] responses: '200': description: retrieved cutout content: image/fits: schema: type: string format: binary image/png: schema: type: string format: binary '400': description: retrieval failed content: application/json: schema: Error """ instrument = self.get_query_argument("instrument", "ZTF").upper() if instrument not in INSTRUMENTS: raise ValueError("Instrument name not recognised") # allow access to public data only by default selector = {1} for stream in self.associated_user_object.streams: if "ztf" in stream.name.lower(): selector.update(set(stream.altdata.get("selector", []))) selector = list(selector) try: candid = int(self.get_argument("candid")) cutout = self.get_argument("cutout").capitalize() file_format = self.get_argument("file_format", "png").lower() interval = self.get_argument("interval", default=None) stretch = self.get_argument("stretch", default=None) cmap = self.get_argument("cmap", default=None) known_cutouts = ["Science", "Template", "Difference"] if cutout not in known_cutouts: return self.error( f"Cutout {cutout} of {object_id}/{candid} not in {str(known_cutouts)}" ) known_file_formats = ["fits", "png"] if file_format not in known_file_formats: return self.error( f"File format {file_format} of {object_id}/{candid}/{cutout} not in {str(known_file_formats)}" ) normalization_methods = { "asymmetric_percentile": AsymmetricPercentileInterval( lower_percentile=1, upper_percentile=100 ), "min_max": MinMaxInterval(), "zscale": ZScaleInterval(nsamples=600, contrast=0.045, krej=2.5), } if interval is None: interval = "asymmetric_percentile" normalizer = normalization_methods.get( interval.lower(), AsymmetricPercentileInterval(lower_percentile=1, upper_percentile=100), ) stretching_methods = { "linear": LinearStretch, "log": LogStretch, "asinh": AsinhStretch, "sqrt": SqrtStretch, } if stretch is None: stretch = "log" if cutout != "Difference" else "linear" stretcher = stretching_methods.get(stretch.lower(), LogStretch)() if (cmap is None) or ( cmap.lower() not in ["bone", "gray", "cividis", "viridis", "magma"] ): cmap = "bone" else: cmap = cmap.lower() query = { "query_type": "find", "query": { "catalog": "ZTF_alerts", "filter": { "candid": candid, "candidate.programid": {"$in": selector}, }, "projection": {"_id": 0, f"cutout{cutout}": 1}, }, "kwargs": {"limit": 1, "max_time_ms": 5000}, } response = kowalski.query(query=query) if response.get("status", "error") == "success": alert = response.get("data", [dict()])[0] else: return self.error("No cutout found.") cutout_data = bj.loads(bj.dumps([alert[f"cutout{cutout}"]["stampData"]]))[0] # unzipped fits name fits_name = pathlib.Path(alert[f"cutout{cutout}"]["fileName"]).with_suffix( "" ) # unzip and flip about y axis on the server side with gzip.open(io.BytesIO(cutout_data), "rb") as f: with fits.open(io.BytesIO(f.read())) as hdu: header = hdu[0].header data_flipped_y = np.flipud(hdu[0].data) if file_format == "fits": hdu = fits.PrimaryHDU(data_flipped_y, header=header) hdul = fits.HDUList([hdu]) stamp_fits = io.BytesIO() hdul.writeto(fileobj=stamp_fits) self.set_header("Content-Type", "image/fits") self.set_header( "Content-Disposition", f"Attachment;filename={fits_name}" ) self.write(stamp_fits.getvalue()) if file_format == "png": buff = io.BytesIO() plt.close("all") fig, ax = plt.subplots(figsize=(4, 4)) fig.subplots_adjust(0, 0, 1, 1) ax.set_axis_off() # replace nans with median: img = np.array(data_flipped_y) # replace dubiously large values xl = np.greater(np.abs(img), 1e20, where=~np.isnan(img)) if img[xl].any(): img[xl] = np.nan if np.isnan(img).any(): median = float(np.nanmean(img.flatten())) img = np.nan_to_num(img, nan=median) norm = ImageNormalize(img, stretch=stretcher) img_norm = norm(img) vmin, vmax = normalizer.get_limits(img_norm) ax.imshow(img_norm, cmap=cmap, origin="lower", vmin=vmin, vmax=vmax) plt.savefig(buff, dpi=42) buff.seek(0) plt.close("all") self.set_header("Content-Type", "image/png") self.write(buff.getvalue()) except Exception: _err = traceback.format_exc() return self.error(f"failure: {_err}")
result = Observations.query_object('M83') selected_bands = result[(result['obs_collection'] == 'HST') & (result['instrument_name'] == 'WFC3/UVIS') & ((result['filters'] == 'F657N') | (result['filters'] == 'F487N') | (result['filters'] == 'F336W')) & (result['target_name'] == 'MESSIER-083')] prodlist = Observations.get_product_list(selected_bands) filtered_prodlist = Observations.filter_products(prodlist) downloaded = Observations.download_products(filtered_prodlist) blue = fits.open(downloaded['Local Path'][2]) red = fits.open(downloaded['Local Path'][5]) green = fits.open(downloaded['Local Path'][8]) target_header = red['SCI'].header green_repr, _ = reproject.reproject_interp(green['SCI'], target_header) blue_repr, _ = reproject.reproject_interp(blue['SCI'], target_header) rgb_img = make_lupton_rgb( ImageNormalize(vmin=0, vmax=1)(red['SCI'].data), ImageNormalize(vmin=0, vmax=0.3)(green_repr), ImageNormalize(vmin=0, vmax=1)(blue_repr), stretch=0.1, minimum=0, ) plt.imshow(rgb_img, origin='lower', interpolation='none')
def _make_pretty_from_fits(fname=None, title=None, figsize=(10, 10 / 1.325), dpi=150, alpha=0.2, number_ticks=7, clip_percent=99.9, **kwargs): with open_fits(fname) as hdu: header = hdu[0].header data = hdu[0].data data = focus_utils.mask_saturated(data) wcs = WCS(header) if not title: field = header.get('FIELD', 'Unknown field') exptime = header.get('EXPTIME', 'Unknown exptime') filter_type = header.get('FILTER', 'Unknown filter') try: date_time = header['DATE-OBS'] except KeyError: # If we don't have DATE-OBS, check filename for date try: basename = os.path.splitext(os.path.basename(fname))[0] date_time = date_parser.parse(basename).isoformat() except Exception: # Otherwise use now date_time = current_time(pretty=True) date_time = date_time.replace('T', ' ', 1) title = '{} ({}s {}) {}'.format(field, exptime, filter_type, date_time) norm = ImageNormalize(interval=PercentileInterval(clip_percent), stretch=LogStretch()) fig = Figure() FigureCanvas(fig) fig.set_size_inches(*figsize) fig.dpi = dpi if wcs.is_celestial: ax = fig.add_subplot(1, 1, 1, projection=wcs) ax.coords.grid(True, color='white', ls='-', alpha=alpha) ra_axis = ax.coords['ra'] ra_axis.set_axislabel('Right Ascension') ra_axis.set_major_formatter('hh:mm') ra_axis.set_ticks( number=number_ticks, color='white', exclude_overlapping=True ) dec_axis = ax.coords['dec'] dec_axis.set_axislabel('Declination') dec_axis.set_major_formatter('dd:mm') dec_axis.set_ticks( number=number_ticks, color='white', exclude_overlapping=True ) else: ax = fig.add_subplot(111) ax.grid(True, color='white', ls='-', alpha=alpha) ax.set_xlabel('X / pixels') ax.set_ylabel('Y / pixels') im = ax.imshow(data, norm=norm, cmap=palette, origin='lower') fig.colorbar(im) fig.suptitle(title) new_filename = fname.replace('.fits', '.jpg') fig.savefig(new_filename, bbox_inches='tight') # explicitly close and delete figure fig.clf() del fig return new_filename
def show_stamps(pscs, frame_idx=None, stamp_size=11, aperture_position=None, show_residual=False, stretch=None, save_name=None, show_max=False, show_pixel_grid=False, **kwargs): if aperture_position is None: midpoint = (stamp_size - 1) / 2 aperture_position = (midpoint, midpoint) ncols = len(pscs) if show_residual: ncols += 1 nrows = 1 fig = Figure() FigureCanvas(fig) fig.set_figheight(4) fig.set_figwidth(8) if frame_idx is not None: s0 = pscs[0][frame_idx] s1 = pscs[1][frame_idx] else: s0 = pscs[0] s1 = pscs[1] if stretch == 'log': stretch = LogStretch() else: stretch = LinearStretch() norm = ImageNormalize(s0, interval=MinMaxInterval(), stretch=stretch) ax1 = fig.add_subplot(nrows, ncols, 1) im = ax1.imshow(s0, cmap=get_palette(), norm=norm) # create an axes on the right side of ax. The width of cax will be 5% # of ax and the padding between cax and ax will be fixed at 0.05 inch. # https://stackoverflow.com/questions/18195758/set-matplotlib-colorbar-size-to-match-graph divider = make_axes_locatable(ax1) cax = divider.append_axes("right", size="5%", pad=0.05) fig.colorbar(im, cax=cax) ax1.set_title('Target') # Comparison ax2 = fig.add_subplot(nrows, ncols, 2) im = ax2.imshow(s1, cmap=get_palette(), norm=norm) divider = make_axes_locatable(ax2) cax = divider.append_axes("right", size="5%", pad=0.05) fig.colorbar(im, cax=cax) ax2.set_title('Comparison') if show_pixel_grid: add_pixel_grid(ax1, stamp_size, stamp_size, show_superpixel=False) add_pixel_grid(ax2, stamp_size, stamp_size, show_superpixel=False) if show_residual: ax3 = fig.add_subplot(nrows, ncols, 3) # Residual residual = s0 - s1 im = ax3.imshow(residual, cmap=get_palette(), norm=ImageNormalize(residual, interval=MinMaxInterval(), stretch=LinearStretch())) divider = make_axes_locatable(ax3) cax = divider.append_axes("right", size="5%", pad=0.05) fig.colorbar(im, cax=cax) ax3.set_title('Noise Residual') ax3.set_title('Residual RMS: {:.01%}'.format(residual.std())) ax3.set_yticklabels([]) ax3.set_xticklabels([]) if show_pixel_grid: add_pixel_grid(ax1, stamp_size, stamp_size, show_superpixel=False) # Turn off tick labels ax1.set_yticklabels([]) ax1.set_xticklabels([]) ax2.set_yticklabels([]) ax2.set_xticklabels([]) if save_name: try: fig.savefig(save_name) except Exception as e: warn("Can't save figure: {}".format(e)) return fig
def pathtest(step_input_filename, reffile, comparison_filename, writefile=True, show_figs=True, save_figs=False, threshold_diff=1.0e-7, debug=False): """ This function calculates the difference between the pipeline and calculated pathloss values. The functions use the output of sourcetype. Args: step_input_filename: str, name of sourcetype step output fits file reffile: str, path to the pathloss MSA UNI reference fits files comparison_filename: str, path to comparison pipeline pathloss file writefile: boolean, if True writes fits files of calculated pathloss and difference images show_figs: boolean, whether to show plots or not save_figs: boolean, save the plots threshold_diff: float, threshold difference between pipeline output & ESA file debug: boolean, if true print statements will show on-screen Returns: - 1 plot, if told to save and/or show them. - median_diff: Boolean, True if smaller or equal to threshold. - log_msgs: list, all print statements are captured in this variable """ log_msgs = [] # start the timer pathtest_start_time = time.time() # get info from the rate file header det = fits.getval(step_input_filename, "DETECTOR", 0) msg = 'step_input_filename=' + step_input_filename print(msg) log_msgs.append(msg) exptype = fits.getval(step_input_filename, "EXP_TYPE", 0) grat = fits.getval(step_input_filename, "GRATING", 0) filt = fits.getval(step_input_filename, "FILTER", 0) # aper = fits.getval(step_input_filename[1], "SLTNAME", 0) msg = "path_loss_file: Grating:" + grat + " Filter:" + filt + " EXP_TYPE:" + exptype print(msg) log_msgs.append(msg) # get the reference files msg = "Using reference file: " + reffile print(msg) log_msgs.append(msg) if writefile: # create the fits list to hold the calculated flat values for each slit hdu0 = fits.PrimaryHDU() outfile = fits.HDUList() outfile.append(hdu0) # create fits list to hold image of pipeline-calculated diff values hdu0 = fits.PrimaryHDU() compfile = fits.HDUList() compfile.append(hdu0) # list to determine if pytest is passed or not total_test_result = [] print("""Checking if files exist and obtaining datamodels. This takes a few minutes...""") if os.path.isfile(comparison_filename): if debug: print('Comparison file does exist.') else: result_msg = 'Comparison file does NOT exist. Skipping pathloss test.' log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the comparison data model pathloss_pipe = datamodels.open(comparison_filename) if debug: print('got comparison datamodel!') if os.path.isfile(step_input_filename): if debug: print('Input file does exist.') else: result_msg = 'Input file does NOT exist. Skipping pathloss test.' log_msgs.append(result_msg) result = 'skip' return result, result_msg, log_msgs # get the input data model pl = datamodels.open(step_input_filename) if debug: print('got input datamodel!') # loop through the slits msg = " Looping through the slits... " print(msg) log_msgs.append(msg) slit_val = 0 for slit, pipe_slit in zip(pl.slits, pathloss_pipe.slits): slit_val = slit_val + 1 mode = "MOS" is_point_source = False print("Retrieving extensions") ps_uni_ext_list = get_mos_ps_uni_extensions(reffile, is_point_source) slit_id = slit.name try: nshutters = util.get_num_msa_open_shutters(slit.shutter_state) if is_point_source: if nshutters == 3: shutter_key = "MOS1x3" elif nshutters == 1: shutter_key = "MOS1x1" ext = ps_uni_ext_list[0][shutter_key] print("Retrieved point source extension") if is_point_source is False: if nshutters == 1: shutter_key = "MOS1x1" elif nshutters == 3: shutter_key = "MOS1x3" ext = ps_uni_ext_list[1][shutter_key] print("Retrieved extended source extension {}".format(ext)) except KeyError: print("Unable to retrieve extension. Using ext 3, but may be 7") ext = 3 wcs_obj = slit.meta.wcs x, y = wcstools.grid_from_bounding_box(wcs_obj.bounding_box, step=(1, 1), center=True) ra, dec, wave = slit.meta.wcs(x, y) wave_sci = wave * 10**(-6) # microns --> meters plcor_ref_ext = fits.getdata(reffile, ext) print("plcor_ref_ext.shape", plcor_ref_ext.shape) hdul = fits.open(reffile) plcor_ref = hdul[1].data w = wcs.WCS(hdul[1].header) w1, y1, x1 = np.mgrid[:plcor_ref.shape[0], :plcor_ref. shape[1], :plcor_ref.shape[2]] slitx_ref, slity_ref, wave_ref = w.all_pix2world(x1, y1, w1, 0) comp_sci = pipe_slit.data previous_sci = slit.data pipe_correction = pipe_slit.pathloss # set up generals for all the plots font = {'weight': 'normal', 'size': 10} matplotlib.rc('font', **font) corr_vals = np.interp(wave_sci, wave_ref[:, 0, 0], plcor_ref_ext) corrected_array = previous_sci / corr_vals # plots: step_input_filepath = step_input_filename.replace(".fits", "") # my correction values fig = plt.figure() ax = plt.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) ax.get_xaxis().get_major_formatter().set_scientific(False) # calculated correction values plt.subplot(221) norm = ImageNormalize(corr_vals) plt.imshow(corr_vals, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('x in pixels') plt.ylabel('y in pixels') plt.title('Calculated Correction') plt.colorbar() # pipeline correction values plt.subplot(222) norm = ImageNormalize(pipe_correction) plt.imshow(pipe_correction, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('x in pixels') plt.ylabel('y in pixels') plt.title('Pipeline Correction') plt.colorbar() # residuals (pipe correction - my correction) corr_residuals = pipe_correction - corr_vals plt.subplot(223) norm = ImageNormalize(corr_residuals) plt.ticklabel_format(useOffset=False) plt.imshow(corr_residuals, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.xlabel('x in pixels') plt.ylabel('y in pixels') plt.title('Correction residuals') plt.colorbar() # my science data after pathloss plt.subplot(224) norm = ImageNormalize(corrected_array) plt.imshow(corrected_array, norm=norm, aspect=10.0, origin='lower', cmap='viridis') plt.title('Corrected Data After Pathloss') plt.xlabel('x in pixels') plt.ylabel('y in pixels') plt.colorbar() fig.suptitle("MOS UNI Pathloss Calibration Testing") if show_figs: plt.show() if save_figs: plt_name = step_input_filepath + "Pathloss_test_slitlet_" + str( mode) + "_UNI_" + str(slit_id) + ".png" plt.savefig(plt_name) print('Figure saved as: ', plt_name) plt.close() ax = plt.subplot(212) plt.hist(corr_residuals[~np.isnan(corr_residuals)], bins=100, range=(-0.00000013, 0.00000013)) plt.title('Residuals Histogram') plt.xlabel("Correction Value") plt.ylabel("Number of Occurences") nanind = np.isnan(corr_residuals) # get all the nan indexes notnan = ~nanind # get all the not-nan indexes arr_mean = np.mean(corr_residuals[notnan]) arr_median = np.median(corr_residuals[notnan]) arr_stddev = np.std(corr_residuals[notnan]) plt.axvline(arr_mean, label="mean = %0.3e" % (arr_mean), color="g") plt.axvline(arr_median, label="median = %0.3e" % (arr_median), linestyle="-.", color="b") str_arr_stddev = "stddev = {:0.3e}".format(arr_stddev) ax.text(0.73, 0.67, str_arr_stddev, transform=ax.transAxes, fontsize=16) plt.legend() plt.minorticks_on() # Show and/or save figures if save_figs: plt_name = step_input_filepath + "Pathlosstest_MOS_UNI_slitlet_" + slit_id + ".png" plt.savefig(plt_name) print('Figure saved as: ', plt_name) if show_figs: plt.show() elif not save_figs and not show_figs: msg = "Not making plots because both show_figs and save_figs were set to False." if debug: print(msg) log_msgs.append(msg) elif not save_figs: msg = "Not saving plots because save_figs was set to False." if debug: print(msg) log_msgs.append(msg) plt.close() # create fits file to hold the calculated pathloss for each slit if writefile: msg = "Saving the fits files with the calculated pathloss for each slit..." print(msg) log_msgs.append(msg) # this is the file to hold the image of pipeline-calculated difference values outfile_ext = fits.ImageHDU(corr_vals, name=slit_id) outfile.append(outfile_ext) # this is the file to hold the image of pipeline-calculated difference values compfile_ext = fits.ImageHDU(corr_residuals, name=slit_id) compfile.append(compfile_ext) if corr_residuals[~np.isnan(corr_residuals)].size == 0: msg1 = """Unable to calculate statistics because difference array has all values as NaN. Test will be set to FAILED.""" print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: msg = "Calculating statistics... " print(msg) log_msgs.append(msg) # ignore outliers: corr_residuals = corr_residuals[np.where((corr_residuals != 999.0) & (corr_residuals < 0.1) & (corr_residuals > -0.1))] if corr_residuals.size == 0: msg1 = """ * Unable to calculate statistics because difference array has all outlier values. Test will be set to FAILED.""" print(msg1) log_msgs.append(msg1) test_result = "FAILED" else: stats_and_strings = auxfunc.print_stats(corr_residuals, "Difference", float(threshold_diff), abs=True) stats, stats_print_strings = stats_and_strings corr_residuals_mean, corr_residuals_median, corr_residuals_std = stats for msg in stats_print_strings: log_msgs.append(msg) # This is the key argument for the assert pytest function median_diff = False if abs(corr_residuals_median) <= float(threshold_diff): median_diff = True if median_diff: test_result = "PASSED" else: test_result = "FAILED" msg = " *** Result of the test: " + test_result + "\n" print(msg) log_msgs.append(msg) total_test_result.append(test_result) if writefile: outfile_name = step_input_filename.replace( "srctype", det + "_calcuated_FS_UNI_pathloss") compfile_name = step_input_filename.replace( "srctype", det + "_comparison_FS_UNI_pathloss") # create the fits list to hold the calculated flat values for each slit outfile.writeto(outfile_name, overwrite=True) # this is the file to hold the image of pipeline-calculated difference values compfile.writeto(compfile_name, overwrite=True) msg = "\nFits file with calculated pathloss values of each slit saved as: " print(msg) log_msgs.append(msg) print(outfile_name) log_msgs.append(outfile_name) msg = "Fits file with comparison (pipeline pathloss - calculated pathloss) saved as: " print(msg) log_msgs.append(msg) print(compfile_name) log_msgs.append(compfile_name) # If all tests passed then pytest will be marked as PASSED, else it will be FAILED FINAL_TEST_RESULT = False for t in total_test_result: if t == "FAILED": FINAL_TEST_RESULT = False break else: FINAL_TEST_RESULT = True if FINAL_TEST_RESULT: msg = "\n *** Final result for path_loss test will be reported as PASSED *** \n" print(msg) log_msgs.append(msg) result_msg = "All slits PASSED path_loss test." else: msg = "\n *** Final result for path_loss test will be reported as FAILED *** \n" print(msg) log_msgs.append(msg) result_msg = "One or more slits FAILED path_loss test." # end the timer pathloss_end_time = time.time() - pathtest_start_time if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60.0 # in minutes pathloss_tot_time = "* Script msa_uni.py took ", repr( pathloss_end_time) + " minutes to finish." if pathloss_end_time > 60.0: pathloss_end_time = pathloss_end_time / 60. # in hours pathloss_tot_time = "* Script msa_uni.py took ", repr( pathloss_end_time) + " hours to finish." else: pathloss_tot_time = "* Script msa_uni.py took ", repr( pathloss_end_time) + " seconds to finish." print(pathloss_tot_time) log_msgs.append(pathloss_tot_time) return FINAL_TEST_RESULT, result_msg, log_msgs
def main(): #Load in file, need to change name to file desired flatname = 'customsci237' filein = fits.open("rg" + flatname + ".fits") #initialize data array data = np.zeros((12, 4176, 512)) #fill data array for i in range(12): data[i] = filein[i + 1].data print(data.shape) #import the corresponding fiber-bundle gap mask that defines where the gaps are gaps = np.genfromtxt("blkmask_xS20180408S0238", dtype='int', unpack=True)[2:, :] print(gaps.shape) # corr=fits.open("brgS20180414S0139.fits") # # plt.figure # # plt.imshow(data[0]-corr[1].data) # # plt.show() # for i in range(len(gaps[0])): # for j in range(12): # # print("ccd: "+str(j)) # amp=corr[j+1].data # print(amp.shape) # if j==0: # fibergap=np.median(amp[gaps[0,i]:gaps[1,i],:],axis=0) # # if i==0: # # plt.figure # # plt.imshow(amp[gaps[0,i]:gaps[1,i],:20]) # # plt.show() # # print(fibergap[:10]) # else: # fibergap=np.hstack((fibergap,np.median(amp[gaps[0,i]:gaps[1,i],:],axis=0))) # print(fibergap.shape) # plt.figure # plt.plot(np.arange(6144),fibergap) # plt.show() #initialize scattered light model array x = np.arange(512) y = np.arange(4176) xx, yy = np.meshgrid(x, y) scatteredlight = np.zeros((12, 4176, 512)) * np.nan #Create copy of data array for later testdata = data[0] for i in range(11): testdata = np.hstack((testdata, data[i + 1])) print(testdata.shape) #Show image before scattered light subtraction norm = ImageNormalize(testdata, interval=ZScaleInterval()) plt.figure plt.imshow(testdata, norm=norm) plt.show() #Define order of fitting in x direction. Can be higher due to having many points with full coverage of axis xorder = 3 #xposition of any pixel to be used for fitting later xv = np.arange(512 * 4) #initialize variables to be filled in later #there are 15 gaps, 3 ccds, and each ccd has 4 amps, each with a width of 512 pixels polyfits = np.zeros((15, 3, xorder + 1)) ccds = np.zeros((3, 15, 512 * 4)) yerr = np.zeros((3, 15, 512 * 4)) #There are 15 gaps in every image used here, initialize y position varibale for these gaps yv = np.zeros(15) #Save all fits to a pdf to be examined later if needed. with PdfPages('guanolight_xfits.pdf') as pdf: #Iterate over all gaps (in this case 15) for i in range(len(gaps[0])): #Measure the y position of the gap we are on yv[i] = np.median(np.arange(gaps[1, i] - gaps[0, i]) + gaps[0, i]) #There are 3 CCDs next to each other in the x axis. Iterate over to fit each separately for j in range(3): #initialize the cleaned total CCD array cleanx = np.zeros((4176, 512 * 4)) * np.nan #4 amps per CCD, each is it's own extension in the fits file. Here we manually clean hot columns of pixels that are constant in all images #and group each CCD amps together into one cleaned array. for k in range(4): #get overall amp number (0-11) l = j * 4 + k print("amp: " + str(l)) #temporary data variable test = data[l] #upper and lower x indicies that this amp covers on the CCD ux, lx = 512 * (k + 1), 512 * k #initialize cleaned amp data to be filled in clean = np.zeros((gaps[1, i] - gaps[0, i], 512)) * np.nan #All of these if, elif statements target hot pixels that are constant and are caught by the manual mask in @remove_outliers function. #These pixels are removed first before removing outliers from other sources in the else. if l == 0: clean[:, 53:] = remove_outliers( test[gaps[0, i]:gaps[1, i], 53:], 4) elif l == 3: clean[:, :497] = remove_outliers( test[gaps[0, i]:gaps[1, i], :497], 4) elif l == 4: temp = test[gaps[0, i]:gaps[1, i], :] temp[:, 239:243] = np.nan clean[:, 16:] = remove_outliers(temp[:, 16:], 4) print(clean.shape) elif l == 7: clean[:, :489] = remove_outliers( test[gaps[0, i]:gaps[1, i], :489], 4) elif l == 8: clean[:, 17:] = remove_outliers( test[gaps[0, i]:gaps[1, i], 17:], 4) elif l == 11: clean[:, :489] = remove_outliers( test[gaps[0, i]:gaps[1, i], :489], 4) else: clean = remove_outliers(test[gaps[0, i]:gaps[1, i], :], 4) #input the cleaned amp data into the cleaned CCD array cleanx[gaps[0, i]:gaps[1, i], lx:ux] = clean #condense each gap down to a single point along it's y axis extent to create a single line along x ynew = np.median(cleanx[gaps[0, i]:gaps[1, i], :], axis=0) #only fit to points that aren't nans mask = ~np.isnan(ynew) xfit = np.polyfit(xv[mask], ynew[mask], xorder) #save the polynomial fit parameters for later polyfits[i, j, :] = xfit #use difference from fit to actual image within the gap for error estimate yfit = np.poly1d(xfit) yerr[j, i, :] = np.ones( len(xv)) * np.std(ynew[mask] - yfit(xv[mask])) #save fit across ccd for the y fits later ccds[j, i, :] = yfit(xv) fig, ax = plt.subplots() ax.plot(xv[mask], ynew[mask]) ax.plot(xv, ynew) ax.plot(xv, yfit(xv)) ax.set_ylim(-10., 30.) if (j == 0) & (i == 0): plt.show() pdf.savefig(fig) plt.close() #Set y axis fit order. Have to be careful with this as we will essentially be fitting each column of pixels using 15 points yorder = 3 #initialize y-axis for fitting and the resulting scattered light model ynew = np.arange(4176) modelscatlight = np.zeros((4176, 512 * 12)) #again saving fits in pdf with PdfPages('guanolight_yfits.pdf') as pdf: #iterating over 3 CCDs for i in range(3): #iterating over the columns in groups of 4 columns median'd together along the x axis for j in range(512): #initial x-position of column to be fit xpos = j * 4 + i * 512 * 4 #median 4 columns together to be fit z = np.median(ccds[i, :, j * 4:(j + 1) * 4], axis=-1) #pull errors from x-axis fits err = yerr[i, :, j * 4] # z=np.hstack((z,z[-1])) # print(np.median(testdata[0:10,i*512*4+j],axis=0)) #fit along y-axis (sorry for poor variable naming). Weight each point in fit by the inverse of the error. zfit = np.polyfit(yv, z, yorder, w=err**(-1.)) znew = np.poly1d(zfit) #save the fit for those 4 columns into the model modelscatlight[:, xpos:xpos + 4] = np.array( [znew(ynew), znew(ynew), znew(ynew), znew(ynew)]).T #plot for pdf fig, ax = plt.subplots() ax.plot(ynew, znew(ynew), 'r', linewidth=2.0) ax.errorbar(yv, z, yerr=err, fmt='o', color='b', ms=4.0) pdf.savefig(fig) #show first plot for sanity check if (i == 0) & (j == 0): plt.show() plt.close() #plot total model after all fitting plt.figure plt.imshow(modelscatlight) plt.show() #subtract the model from the data over all 12 amps for i in range(12): filein[i + 1].data = data[i] - modelscatlight[:, 512 * i:512 * (i + 1)] #here we examine all the fiber-bundle gaps post subtraction to see how close to zero they become. If perfectly subtracted then they should all show zero flux. #iterate over each gap for i in range(len(gaps[0])): plt.figure #iterate over each amp for j in range(12): # print("ccd: "+str(j)) amp = filein[j + 1].data print(amp.shape) #if first time then initialize variable otherwise add new amp to variable if j == 0: fibergap = np.median(amp[gaps[0, i]:gaps[1, i], :], axis=0) else: fibergap = np.hstack( (fibergap, np.median(amp[gaps[0, i]:gaps[1, i], :], axis=0))) print(fibergap.shape) #plot gap spanning all amps plt.plot(np.arange(6144), fibergap) plt.show() #write to subtracted image to file. filein.writeto("brgcustomsci237.fits", overwrite=True)
def nice_norm(image: np.ndarray): return ImageNormalize(image, interval=ZScaleInterval(), stretch=SqrtStretch())