def download_hitran(m, i, numin, numax): """ Download HITRAN data for a particular molecule. Based on fetch function from hapi.py. Parameters ---------- m : int HITRAN molecule number i : int HITRAN isotopologue number numin : real lower wavenumber bound numax : real upper wavenumber bound """ iso_id = str(ISO[(m, i)][ISO_INDEX["id"]]) mol_name = ISO[(m, i)][ISO_INDEX["mol_name"]] filename = os.path.join(cache_location, "{0}.data".format(mol_name)) CHUNK = 64 * 1024 data = dict(iso_ids_list=iso_id, numin=numin, numax=numax) with open(filename, "w") as fp: response = commons.send_request(HITRAN_URL, data, 10, request_type="GET") if "Content-Length" in response.headers: total_length = response.headers.get("Content-Length") pb = ProgressBar(int(total_length)) for chunk in response.iter_content(chunk_size=CHUNK): fp.write(chunk.decode("utf-8")) try: pb.update(CHUNK) except NameError: pass
def _download_file(self, url, local_filepath, timeout=None, auth=None): """ Download a file. Resembles `astropy.utils.data.download_file` but uses the local ``_session`` """ response = self._session.get(url, timeout=timeout, stream=True, auth=auth) if 'content-length' in response.headers: length = int(response.headers['content-length']) else: length = 1 pb = ProgressBar(length) blocksize = astropy.utils.data.conf.download_block_size bytes_read = 0 with open(local_filepath, 'wb') as f: for block in response.iter_content(blocksize): f.write(block) bytes_read += blocksize pb.update(bytes_read if bytes_read <= length else length) response.close()
def extract_poly_slice(cube, polygons, width=1.0): nx = len(polygons) nz = cube.shape[0] slice = np.zeros((nz, nx)) p = ProgressBar(len(polygons)) for i, polygon in enumerate(polygons): p.update() # Find bounding box bbxmin = int(round(np.min(polygon.x))-1) bbxmax = int(round(np.max(polygon.x))+2) bbymin = int(round(np.min(polygon.y))-1) bbymax = int(round(np.max(polygon.y))+2) # Loop through pixels that might overlap for xmin in np.arange(bbxmin, bbxmax): for ymin in np.arange(bbymin, bbymax): area = square_polygon_overlap_area(xmin-0.5, xmin+0.5, ymin-0.5, ymin+0.5, polygon.x, polygon.y) if area > 0: slice[:, i] += cube[:, ymin, xmin] * area print("") return slice
def ratio_to_dens_slow(ratio, c11, c22): """ Shape: ratio [z,y,x] c11 [y,x] c22 [y,x] """ assert c11.size == c22.size == ratio[0,:,:].size fshape = [ratio.shape[0], ratio.shape[1]*ratio.shape[2]] rrs = ratio.reshape(fshape).T outc = (ratio*0).reshape(fshape) + np.nan # set up a grid... pb = ProgressBar((c11.size)) for ii,(r,c1,c2) in enumerate(zip(rrs, c11.flat, c22.flat)): #print r.shape,c1,c2 if np.isfinite(c1) and np.isfinite(c2) and np.any(np.isfinite(r)): tauratio, ok = get_tau_ratio(c1,c2) inds = np.argsort(tauratio[ok]) outc[:,ii] = np.interp(r, tauratio[ok][inds], dens[ok][inds], np.nan, np.nan) pb.update() #pb.finish() return outc.reshape(ratio.shape)
def fit_ch3cn_lines(spectra, save_prefix, velo=56*u.km/u.s, ampguess=-0.01): all_ch3cn = table.vstack([ch3cn, ch3cn_v]) all_ch3cn.add_column(table.Column(name='FittedAmplitude', data=np.zeros(len(all_ch3cn)))) all_ch3cn.add_column(table.Column(name='FittedCenter', data=np.zeros(len(all_ch3cn)))) all_ch3cn.add_column(table.Column(name='FittedWidth', data=np.zeros(len(all_ch3cn)))) all_ch3cn.add_column(table.Column(name='FittedAmplitudeError', data=np.zeros(len(all_ch3cn)))) all_ch3cn.add_column(table.Column(name='FittedCenterError', data=np.zeros(len(all_ch3cn)))) all_ch3cn.add_column(table.Column(name='FittedWidthError', data=np.zeros(len(all_ch3cn)))) vkms = velo.to(u.km/u.s).value pl.figure(1).clf() ax = pl.gca() pb = ProgressBar(len(spectra) * len(all_ch3cn)) ii = 0 for sp in spectra: sp.xarr.convert_to_unit(u.GHz) mid = np.median(sp.data) for line in all_ch3cn: frq = line['Freq-GHz']*u.GHz if sp.xarr.in_range(frq*(1-velo/constants.c)): offset = ii*0.000 + mid ii += 1 sp.xarr.convert_to_unit(u.km/u.s, refX=frq) sp.plotter(axis=ax, clear=False, offset=offset) sp.specfit(fittype='vheightgaussian', guesses=[mid, ampguess, vkms, 2],) line['FittedAmplitude'] = sp.specfit.parinfo['AMPLITUDE0'].value line['FittedCenter'] = sp.specfit.parinfo['SHIFT0'].value line['FittedWidth'] = sp.specfit.parinfo['WIDTH0'].value line['FittedAmplitudeError'] = sp.specfit.parinfo['AMPLITUDE0'].error line['FittedCenterError'] = sp.specfit.parinfo['SHIFT0'].error line['FittedWidthError'] = sp.specfit.parinfo['WIDTH0'].error sp.xarr.convert_to_unit(u.GHz) pb.update() pl.xlim(vkms-14, vkms+14) #pl.ylim(0, offset) pl.draw() pl.show() pl.savefig(save_prefix+"_spectra_overlay.png") pl.figure(2).clf() pl.plot(all_ch3cn['E_U (K)'], all_ch3cn['FittedWidth'], 'o') pl.xlabel("E$_U$ (K)") pl.ylabel("$\sigma$ (km/s)") pl.ylim(0,3.5) pl.savefig(save_prefix+"_sigma_vs_eupper.png") pl.figure(3).clf() pl.plot(all_ch3cn['E_U (K)'], all_ch3cn['FittedCenter'], 'o') pl.xlabel("E$_U$ (K)") pl.ylabel("$v_{lsr}$ (km/s)") pl.ylim(vkms-3, vkms+3) pl.savefig(save_prefix+"_vcen_vs_eupper.png")
def spectral_regrid(cube, outgrid): """ Spectrally regrid a cube onto a new spectral output grid (this is apparently redundant with regrid_cube_hdu) """ assert isinstance(cube, SpectralCube) inaxis = cube.spectral_axis.to(outgrid.unit) indiff = np.mean(np.diff(inaxis)) outdiff = np.mean(np.diff(outgrid)) if outdiff < 0: outgrid=outgrid[::-1] outdiff = np.mean(np.diff(outgrid)) if indiff < 0: cubedata = cube.filled_data[::-1] inaxis = cube.spectral_axis.to(outgrid.unit)[::-1] indiff = np.mean(np.diff(inaxis)) else: cubedata = cube.filled_data[:] if indiff < 0 or outdiff < 0: raise ValueError("impossible.") assert np.all(np.diff(outgrid) > 0) assert np.all(np.diff(inaxis) > 0) np.testing.assert_allclose(np.diff(outgrid), outdiff, err_msg="Output grid must be linear") if outdiff > 2 * indiff: raise ValueError("Input grid has too small a spacing. It needs to be " "smoothed prior to resampling.") newcube = np.empty([outgrid.size, cube.shape[1], cube.shape[2]]) yy,xx = np.indices(cube.shape[1:]) pb = ProgressBar(xx.size) for ix, iy in (zip(xx.flat, yy.flat)): newcube[:,iy,ix] = np.interp(outgrid.value, inaxis.value, cubedata[:,iy,ix].value) pb.update() newheader = cube.header newheader['CRPIX3'] = 1 newheader['CRVAL3'] = outgrid[0].value newheader['CDELT3'] = outdiff.value newheader['CUNIT3'] = outgrid.unit.to_string('FITS') return fits.PrimaryHDU(data=newcube, header=newheader)
def download_file(url, outdir=rawpath): r = requests.get(url, verify=False, stream=True) _, cdisp = cgi.parse_header(r.headers['content-disposition']) outfilename = cdisp['filename'] fullname = os.path.join(outdir, outfilename) pb = ProgressBar(int(r.headers['content-length'])) with open(fullname, 'wb') as f: for chunk in r.iter_content(chunk_size=1024): f.write(chunk) f.flush() pb.update(pb._current_value + 1024) return fullname
def fit_all_tex(xaxis, cube, cubefrequencies, indices, degeneracies, ecube=None, replace_bad=False): """ Parameters ---------- replace_bad : bool Attempt to replace bad (negative) values with their upper limits? """ tmap = np.empty(cube.shape[1:]) Nmap = np.empty(cube.shape[1:]) yy,xx = np.indices(cube.shape[1:]) pb = ProgressBar(xx.size) count=0 for ii,jj in (zip(yy.flat, xx.flat)): if any(np.isnan(cube[:,ii,jj])): tmap[ii,jj] = np.nan else: if replace_bad: uplims = nupper_of_kkms(replace_bad, cubefrequencies, einsteinAij[indices], degeneracies,).value else: uplims = None nuppers = nupper_of_kkms(cube[:,ii,jj], cubefrequencies, einsteinAij[indices], degeneracies, ) if ecube is not None: nupper_error = nupper_of_kkms(ecube[:,ii,jj], cubefrequencies, einsteinAij[indices], degeneracies,).value uplims = 3 * nupper_error if replace_bad: raise ValueError("replace_bad is ignored now...") else: nupper_error = None fit_result = fit_tex(xaxis, nuppers.value, errors=nupper_error, uplims=uplims) tmap[ii,jj] = fit_result[1].value Nmap[ii,jj] = fit_result[0].value pb.update(count) count+=1 return tmap,Nmap
def extract_poly_slice(cube, polygons): nx = len(polygons) nz = cube.shape[0] total_slice = np.zeros((nz, nx)) total_area = np.zeros((nz, nx)) p = ProgressBar(len(polygons)) for i, polygon in enumerate(polygons): p.update() # Find bounding box bbxmin = int(round(np.min(polygon.x)) - 1) bbxmax = int(round(np.max(polygon.x)) + 2) bbymin = int(round(np.min(polygon.y)) - 1) bbymax = int(round(np.max(polygon.y)) + 2) # Clip to cube box bbxmin = max(bbxmin, 0) bbxmax = min(bbxmax, cube.shape[2]) bbymin = max(bbymin, 0) bbymax = min(bbymax, cube.shape[1]) # Loop through pixels that might overlap for xmin in np.arange(bbxmin, bbxmax): for ymin in np.arange(bbymin, bbymax): area = square_polygon_overlap_area(xmin - 0.5, xmin + 0.5, ymin - 0.5, ymin + 0.5, polygon.x, polygon.y) if area > 0: total_slice[:, i] += cube[:, ymin, xmin] * area total_area[:, i] += area total_slice[total_area == 0.0] = np.nan total_slice[total_area > 0.0] /= total_area[total_area > 0.0] print("") return total_slice
def _HEADER_data_size(self, files): """ Given a list of file URLs, return the data size. This is useful for assessing how much data you might be downloading! (This is discouraged by the ALMA archive, as it puts unnecessary load on their system) """ totalsize = 0 * u.B data_sizes = {} pb = ProgressBar(len(files)) for ii, fileLink in enumerate(files): response = self._request('HEAD', fileLink, stream=False, cache=False, timeout=self.TIMEOUT) filesize = (int(response.headers['content-length']) * u.B).to(u.GB) totalsize += filesize data_sizes[fileLink] = filesize log.debug("File {0}: size {1}".format(fileLink, filesize)) pb.update(ii + 1) response.raise_for_status() return data_sizes, totalsize.to(u.GB)
def fit_all_tex(xaxis, cube, cubefrequencies, degeneracies, einsteinAij, errorcube=None, replace_bad=False): """ Parameters ---------- replace_bad : bool Attempt to replace bad (negative) values with their upper limits? """ tmap = np.empty(cube.shape[1:]) Nmap = np.empty(cube.shape[1:]) yy,xx = np.indices(cube.shape[1:]) pb = ProgressBar(xx.size) count=0 for ii,jj in (zip(yy.flat, xx.flat)): if any(np.isnan(cube[:,ii,jj])): tmap[ii,jj] = np.nan else: if replace_bad: neg = cube[:,ii,jj] <= 0 cube[neg,ii,jj] = replace_bad nuppers = nupper_of_kkms(cube[:,ii,jj], cubefrequencies, einsteinAij, degeneracies) if errorcube is not None: enuppers = nupper_of_kkms(errorcube[:,ii,jj], cubefrequencies, einsteinAij, degeneracies) fit_result = fit_tex(xaxis, nuppers.value, errors=enuppers.value) tmap[ii,jj] = fit_result[1].value Nmap[ii,jj] = fit_result[0].value pb.update(count) count+=1 return tmap,Nmap
def _fast_reader(index_map, data): """ Use scipy.ndimage.find_objects to quickly identify subsets of the data to increase speed of dendrogram loading """ flux_by_structure, indices_by_structure = {}, {} from scipy import ndimage idxs = np.unique(index_map[index_map > -1]) # ndimage ignores 0 and -1, but we want index 0 object_slices = ndimage.find_objects(index_map + 1) # find_objects returns a tuple that includes many None values that we # need to get rid of. object_slices = [x for x in object_slices if x is not None] index_cube = np.indices(index_map.shape) # Need to have same length, otherwise assumptions above are wrong assert len(idxs) == len(object_slices) log.debug("Creating index maps for {0} indices...".format(len(idxs))) p = ProgressBar(len(object_slices)) for idx, sl in zip(idxs, object_slices): match = index_map[sl] == idx sl2 = (slice(None),) + sl match_inds = index_cube[sl2][:, match] coords = list(zip(*match_inds)) dd = data[sl][match].tolist() flux_by_structure[idx] = dd indices_by_structure[idx] = coords p.update() return flux_by_structure, indices_by_structure
class Pool(object): """ The main pool object. Manages a set of specified workers. Usage:: commands = [ 'ls -al', 'cd /tmp && mkdir foo', 'date', 'echo "Hello There."', 'sleep 2 && echo "Done."' ] lil = Pool(workers=2) lil.run(commands) Optionally accepts a ``workers`` kwarg. Default is 1. Optionally accepts a ``debug`` kwarg. Default is False. Optionally accepts a ``wait_time`` kwarg. Default is 0.1. """ def __init__(self, workers=1, debug=False, wait_time=0.1): if workers < 1: raise NotEnoughWorkers("You need to use at least one worker.") self.workers = workers self.pool = {} self.commands = [] self.callback = None self.debug = debug self.wait_time = wait_time self.progressbar = None def init_progressbar(self): """ Initialise the progress bar. This only happens if run command is called with ``progressbar=True``. """ self.progressbar = ProgressBar(self.command_count()) def prepare_commands(self, commands): """ A hook to override how the commands are added. By default, simply copies the provided command ``list`` to the internal ``commands`` list. """ # Make a copy of the commands to run. self.commands = commands[:] def command_count(self): """ Returns the number of commands to be run. Useful as a hook if you use a different structure for the commands. """ return len(self.commands) def next_command(self): """ Fetches the next command for processing. Will return ``None`` if there are no commands remaining (unless ``Pool.debug = True``). """ try: return self.commands.pop(0) except IndexError: if self.debug: raise return None def process_kwargs(self, command): """ A hook to alter the kwargs given to ``subprocess.Process``. Takes a ``command`` argument, which is unused by default, but can be used to switch the flags used. By default, only specifies ``shell=True``. """ return {"shell": True} def create_process(self, command): """ Given a provided command (string or list), creates a new process to execute the command. """ logging.debug("Starting process to handle command '%s'." % command) kwargs = self.process_kwargs(command) return subprocess.Popen(command, **kwargs) def set_callback(self, callback=None): """ Sets up a callback to be run whenever a process finishes. If called with ``None`` or without any args, it will clear any existing callback. """ self.callback = callback def add_to_pool(self, proc): """ Adds a process to the pool. """ logging.debug("Adding %s to the pool." % proc.pid) self.pool[proc.pid] = proc def remove_from_pool(self, pid): """ Removes a process to the pool. Fails silently if the process id is no longer present (unless ``Pool.debug = True``). """ try: logging.debug("Removing %s from the pool" % pid) del (self.pool[pid]) except KeyError: if self.debug: raise def inspect_pool(self): """ A hook for inspecting the pool's current status. By default, simply makes a log message and returns the length of the pool. """ # Call ``len()`` just once. pool_size = len(self.pool) logging.debug("Current pool size: %s" % pool_size) return pool_size def busy_wait(self): """ A hook to control how often the busy-wait loop runs. By default, sleeps for 0.1 seconds. """ time.sleep(self.wait_time) def run(self, commands=None, callback=None, progressbar=False): """ The method to actually execute all the commands with the pool. Optionally accepts a ``commands`` kwarg, as a shortcut not to have to call ``Pool.prepare_commands``. """ if commands is not None: self.prepare_commands(commands) if callback is not None: self.set_callback(callback) if progressbar is True: self.init_progressbar() keep_running = True while keep_running: self.inspect_pool() if len(self.pool) <= min(self.command_count(), self.workers): command = self.next_command() if not command: self.busy_wait() continue proc = self.create_process(command) self.add_to_pool(proc) # Go in reverse order so offsets never get screwed up. for pid in self.pool.keys(): logging.debug("Checking status on %s" % self.pool[pid].pid) if self.pool[pid].poll() >= 0: if self.callback: self.callback(self.pool[pid]) if progressbar: self.progressbar.update() self.remove_from_pool(pid) keep_running = self.command_count() or len(self.pool) > 0 self.busy_wait() if progressbar: self.progressbar.__exit__(None, None, None)
multigauss_uncerts = np.zeros((3 * max_gauss_comps, ) + peaktemp.shape) multigauss_comps = np.zeros(peaktemp.shape) multigauss_model_cube = np.zeros(subcube.shape) fit_bics = np.zeros((2, ) + peaktemp.shape) show_plots = False # show_plots = True pbar = ProgressBar(len(mask_positions[0])) for y, x in zip(mask_positions[0], mask_positions[1]): pbar.update() # print(f"{y}, {x}") spec = subcube[:, y, x].with_spectral_unit(u.km / u.s) # Fit that spectrum. thickHI_fit, vels, thickHI_fit_model = \ fit_isoturbHI_model_simple(spec.spectral_axis, # [spec_mask], spec, # [spec_mask], peakvels[y, x], err=noise_val, delta_vcent=10 * u.km / u.s, verbose=show_plots, plot_fit=show_plots, use_emcee=False,
def compute_bispectrum(self, show_progress=True, use_pyfftw=False, threads=1, nsamples=100, seed=1000, mean_subtract=False, **pyfftw_kwargs): ''' Do the computation. Parameters ---------- show_progress : optional, bool Show progress bar while sampling the bispectrum. use_pyfftw : bool, optional Enable to use pyfftw, if it is installed. threads : int, optional Number of threads to use in FFT when using pyfftw. nsamples : int, optional Sets the number of samples to take at each vector magnitude. seed : int, optional Sets the seed for the distribution draws. mean_subtract : bool, optional Subtract the mean from the data before computing. This removes the "zero frequency" (i.e., constant) portion of the power, resulting in a loss of phase coherence along the k_1=k_2 line. pyfft_kwargs : Passed to `~turbustat.statistics.rfft_to_fft.rfft_to_fft`. See `here <https://hgomersall.github.io/pyFFTW/pyfftw/interfaces/interfaces.html#interfaces-additional-args>`_ for a list of accepted kwargs. ''' if mean_subtract: norm_data = self.data - self.data.mean() else: norm_data = self.data if use_pyfftw: if PYFFTW_FLAG: if pyfftw_kwargs.get('threads') is not None: pyfftw_kwargs.pop('threads') fftarr = fft2(norm_data, threads=threads, **pyfftw_kwargs) else: warn("pyfftw not installed. Reverting to using numpy.") use_pyfftw = False if not use_pyfftw: fftarr = np.fft.fft2(norm_data) conjfft = np.conj(fftarr) bispec_shape = (int(self.shape[0] / 2.), int(self.shape[1] / 2.)) self._bispectrum = np.zeros(bispec_shape, dtype=np.complex) self._bicoherence = np.zeros(bispec_shape, dtype=np.float) self._tracker = np.zeros(self.shape, dtype=np.int16) biconorm = np.ones_like(self.bispectrum, dtype=float) if show_progress: bar = ProgressBar(np.prod(fftarr.shape) / 4.) prod = product(range(int(fftarr.shape[0] / 2.)), range(int(fftarr.shape[1] / 2.))) with NumpyRNGContext(seed): for n, (k1mag, k2mag) in enumerate(prod): phi1 = ra.uniform(0, 2 * np.pi, nsamples) phi2 = ra.uniform(0, 2 * np.pi, nsamples) k1x = np.asarray([int(k1mag * np.cos(angle)) for angle in phi1]) k2x = np.asarray([int(k2mag * np.cos(angle)) for angle in phi2]) k1y = np.asarray([int(k1mag * np.sin(angle)) for angle in phi1]) k2y = np.asarray([int(k2mag * np.sin(angle)) for angle in phi2]) k3x = np.asarray([int(k1mag * np.cos(ang1) + k2mag * np.cos(ang2)) for ang1, ang2 in zip(phi1, phi2)]) k3y = np.asarray([int(k1mag * np.sin(ang1) + k2mag * np.sin(ang2)) for ang1, ang2 in zip(phi1, phi2)]) samps = fftarr[k1x, k1y] * fftarr[k2x, k2y] * conjfft[k3x, k3y] self._bispectrum[k1mag, k2mag] = np.sum(samps) biconorm[k1mag, k2mag] = np.sum(np.abs(samps)) # Track where we're sampling from in fourier space self._tracker[k1x, k1y] += 1 self._tracker[k2x, k2y] += 1 self._tracker[k3x, k3y] += 1 if show_progress: bar.update(n + 1) self._bicoherence = (np.abs(self.bispectrum) / biconorm) self._bispectrum_amp = np.log10(np.abs(self.bispectrum))
def reject(imfile, catfile, threshold): """Reject noisy detections. Parameters ---------- imfile : str The path to the radio image file catfile : str The path to the source catalog, as obtained from detect.py threshold : float The signal-to-noise threshold below which sources are rejected """ # Extract information from filename outfile = os.path.basename(catfile).split('cat_')[1].split('.dat')[0] region = outfile.split('region')[1].split('_band')[0] band = outfile.split('band')[1].split('_val')[0] min_value = outfile.split('val')[1].split('_delt')[0] min_delta = outfile.split('delt')[1].split('_pix')[0] min_npix = outfile.split('pix')[1] print("\nSource rejection for region {} in band {}".format(region, band)) print("Loading image file") contfile = fits.open(imfile) data = contfile[0].data.squeeze() mywcs = wcs.WCS(contfile[0].header).celestial catalog = Table(Table.read(catfile, format='ascii'), masked=True) beam = radio_beam.Beam.from_fits_header(contfile[0].header) pixel_scale = np.abs( mywcs.pixel_scale_matrix.diagonal().prod())**0.5 * u.deg ppbeam = (beam.sr / (pixel_scale**2)).decompose().value data = data / ppbeam # Remove existing region files if os.path.isfile('./reg/reg_' + outfile + '_annulus.reg'): os.remove('./reg/reg_' + outfile + '_annulus.reg') if os.path.isfile('./reg/reg_' + outfile + '_filtered.reg'): os.remove('./reg/reg_' + outfile + '_filtered.reg') # Load in manually accepted and rejected sources override_accepted = [] override_rejected = [] if os.path.isfile('./.override/accept_' + outfile + '.txt'): override_accepted = np.loadtxt('./.override/accept_' + outfile + '.txt').astype('int') if os.path.isfile('./.override/reject_' + outfile + '.txt'): override_rejected = np.loadtxt('./.override/reject_' + outfile + '.txt').astype('int') print("\nManually accepted sources: ", set(override_accepted)) print("Manually rejected sources: ", set(override_rejected)) print('\nCalculating RMS values within aperture annuli') pb = ProgressBar(len(catalog)) data_cube = [] masks = [] rejects = [] snr_vals = [] mean_backgrounds = [] for i in range(len(catalog)): x_cen = catalog['x_cen'][i] * u.deg y_cen = catalog['y_cen'][i] * u.deg major_fwhm = catalog['major_fwhm'][i] * u.deg minor_fwhm = catalog['minor_fwhm'][i] * u.deg position_angle = catalog['position_angle'][i] * u.deg dend_flux = catalog['dend_flux_band{}'.format(band)][i] annulus_width = 1e-5 * u.deg center_distance = 1e-5 * u.deg # Define some ellipse properties in pixel coordinates position = coordinates.SkyCoord(x_cen, y_cen, frame='icrs', unit=(u.deg, u.deg)) pix_position = np.array(position.to_pixel(mywcs)) pix_major_fwhm = major_fwhm / pixel_scale pix_minor_fwhm = minor_fwhm / pixel_scale # Cutout section of the image we care about, to speed up computation time size = (center_distance + annulus_width + major_fwhm) * 2.2 cutout = Cutout2D(data, position, size, mywcs, mode='partial') cutout_center = regions.PixCoord(cutout.center_cutout[0], cutout.center_cutout[1]) # Define the aperture regions needed for SNR ellipse_reg = regions.EllipsePixelRegion( cutout_center, pix_major_fwhm * 2., pix_minor_fwhm * 2., angle=position_angle ) # Make sure you're running the dev version of regions, otherwise the position angles will be in radians! innerann_reg = regions.CirclePixelRegion( cutout_center, center_distance / pixel_scale + pix_major_fwhm) outerann_reg = regions.CirclePixelRegion( cutout_center, center_distance / pixel_scale + pix_major_fwhm + annulus_width / pixel_scale) # Make masks from aperture regions ellipse_mask = mask(ellipse_reg, cutout) annulus_mask = mask(outerann_reg, cutout) - mask(innerann_reg, cutout) # Plot annulus and ellipse regions data_cube.append(cutout.data) masks.append([annulus_mask, ellipse_mask]) # Calculate the SNR and aperture flux sums bg_rms = rms(cutout.data[annulus_mask.astype('bool')]) peak_flux = np.max(cutout.data[ellipse_mask.astype('bool')]) flux_rms_ratio = peak_flux / bg_rms snr_vals.append(flux_rms_ratio) # Reject bad sources below some SNR threshold rejected = False if flux_rms_ratio <= threshold: rejected = True # Process manual overrides if catalog['_idx'][i] in override_accepted: rejected = False if catalog['_idx'][i] in override_rejected: rejected = True rejects.append(int(rejected)) # Add non-rejected source ellipses to a new region file fname = './reg/reg_' + outfile + '_filtered.reg' with open(fname, 'a') as fh: if os.stat(fname).st_size == 0: fh.write("icrs\n") if not rejected: fh.write("ellipse({}, {}, {}, {}, {}) # text={{{}}}\n".format( x_cen.value, y_cen.value, major_fwhm.value, minor_fwhm.value, position_angle.value, i)) pb.update() # Plot the grid of sources plot_grid(data_cube, masks, rejects, snr_vals, catalog['_idx']) plt.suptitle( 'region={}, band={}, min_value={}, min_delta={}, min_npix={}, threshold={:.4f}' .format(region, band, min_value, min_delta, min_npix, threshold)) plt.show(block=False) # Get overrides from user print( 'Manual overrides example: type "r319, a605" to manually reject source #319 and accept source #605.' ) overrides = input( "\nType manual override list, or press enter to continue:\n").split( ', ') accepted_list = [ s[1:] for s in list(filter(lambda x: x.startswith('a'), overrides)) ] rejected_list = [ s[1:] for s in list(filter(lambda x: x.startswith('r'), overrides)) ] # Save the manually accepted and rejected sources fname = './.override/accept_' + outfile + '.txt' with open(fname, 'a') as fh: for num in accepted_list: fh.write('\n' + str(num)) fname = './.override/reject_' + outfile + '.txt' with open(fname, 'a') as fh: for num in rejected_list: fh.write('\n' + str(num)) print( "Manual overrides written to './.override/' and saved to source catalog. New overrides will be displayed the next time the rejection script is run." ) # Process the new overrides, to be saved into the catalog rejects = np.array(rejects) acc = np.array([a[-2:] for a in accepted_list], dtype=int) rej = np.array([r[-2:] for r in rejected_list], dtype=int) rejects[acc] = 0 rejects[rej] = 1 # Save the catalog with new columns for SNR catalog.add_column(Column(snr_vals), name='snr_band' + band) catalog.add_column(np.invert(catalog.mask['snr_band' + band]).astype(int), name='detected_band' + band) catalog.add_column(Column(rejects), name='rejected') catalog.write('./cat/cat_' + outfile + '_filtered.dat', format='ascii')
def compute_dendro(self, show_progress=False, save_dendro=False, dendro_name=None, dendro_obj=None, periodic_bounds=False): ''' Compute the dendrogram and prune to the minimum deltas. ** min_deltas must be in ascending order! ** Parameters ---------- show_progress : optional, bool Enables the progress bar in astrodendro. save_dendro : optional, bool Saves the dendrogram in HDF5 format. **Requires pyHDF5** dendro_name : str, optional Save name when save_dendro is enabled. ".hdf5" appended automatically. dendro_obj : Dendrogram, optional Input a pre-computed dendrogram object. It is assumed that the dendrogram has already been computed! periodic_bounds : bool, optional Enable when the data is periodic in the spatial dimensions. ''' self._numfeatures = np.empty(self.min_deltas.shape, dtype=int) self._values = [] if dendro_obj is None: if periodic_bounds: # Find the spatial dimensions num_axes = self.data.ndim spat_axes = [] for i, axis_type in enumerate(self._wcs.get_axis_types()): if axis_type["coordinate_type"] == u"celestial": spat_axes.append(num_axes - i - 1) neighbours = periodic_neighbours(spat_axes) else: neighbours = None d = Dendrogram.compute(self.data, verbose=show_progress, min_delta=self.min_deltas[0], min_value=self.dendro_params["min_value"], min_npix=self.dendro_params["min_npix"], neighbours=neighbours) else: d = dendro_obj self._numfeatures[0] = len(d) self._values.append( np.array([struct.vmax for struct in d.all_structures])) if len(self.min_deltas) > 1: # Another progress bar for pruning steps if show_progress: print("Pruning steps.") bar = ProgressBar(len(self.min_deltas[1:])) for i, delta in enumerate(self.min_deltas[1:]): d.prune(min_delta=delta) self._numfeatures[i + 1] = len(d) self._values.append( np.array([struct.vmax for struct in d.all_structures])) if show_progress: bar.update(i + 1)
def __init__(self, SED, Filters, redshift, force_age=True, madau=True, units=u.uJy, verbose=False): """ Parameters ---------- SED : '~smpy.CSP' object Built Filters : '~smpy.FilterSet' object Filter set through which to observe the set of models included in SED object redshift : float of numpy.array Redshift(s) at which models are to be observed force_age : boolean Require age of the stellar population to be younger than the age of the Universe at the desired redshift. madau : boolean Apply IGM absorption following Madau 1999 prescription units : '~astropy.units.Quantity' Desired output units, must be in spectral flux density equivalent verbose : boolean Add additional terminal outputs if true Attributes ---------- fluxes : array Apparent fluxes of CSP models observed through 'Filters' at the desired redshifts AB : array Apparent AB magnitudes of CSP models observed through 'Filters' at the desired redshifts Examples -------- >>> redshifts = np.linspace(0, 3, 10) >>> A = Observe(CSP, Filters, redshifts) """ self.F = Filters self.redshifts = np.array(redshift, ndmin=1) self.wave = SED.wave self.Ms = SED.Ms self.SFR = SED.SFR self.fluxes = np.zeros( np.append([len(self.redshifts), len(self.F.filters)], SED.SED.shape[:-1])) * units self.AB = np.zeros_like(self.fluxes.value) * u.mag self.wl = np.zeros(len(self.F.filters)) * u.AA self.fwhm = np.zeros(len(self.F.filters)) * u.AA self.dl = cosmo.luminosity_distance(self.redshifts).cgs self.dl[self.redshifts == 0] = 10 * c.pc if verbose: bar = ProgressBar(len(self.redshifts)) for i, z in enumerate(self.redshifts): self.lyman_abs = np.ones(len(self.wave)) if madau: self.lyman_abs = np.clip(tau_madau(self.wave, z), 0., 1.) for j, filter in enumerate(self.F.filters): self.wl[j] = filter.lambda_c self.fwhm[j] = filter.fwhm self.fluxes[i, j] = self.calcflux(SED, filter, z, self.dl[i], units) if not force_age: # Set fluxes for ages older than universe to zero agecut = (SED.tg.to(u.Gyr) > cosmo.age(z)) self.fluxes[i, j, :, agecut] = 0. if verbose: assert isinstance(bar, object) bar.update() # Convert spectral flux density to AB magnitudes self.AB = (-2.5 * np.log10(self.fluxes.to(u.Jy) / (3631 * u.Jy))) * u.mag
def compute_surface(self, boundary='continuous', show_progress=True): ''' Computes the SCF up to the given lag value. This is an expensive operation and could take a long time to calculate. Parameters ---------- boundary : {"continuous", "cut"} Treat the boundary as continuous (wrap-around) or cut values beyond the edge (i.e., for most observational data). show_progress : bool, optional Show a progress bar when computing the surface. = ''' if boundary not in ["continuous", "cut"]: raise ValueError("boundary must be 'continuous' or 'cut'.") self._scf_surface = np.zeros((self.size, self.size)) # Convert the lags into pixel units. pix_lags = self._to_pixel(self.roll_lags).value dx = pix_lags.copy() dy = pix_lags.copy() if show_progress: bar = ProgressBar(len(dx) * len(dy)) for n, (x_shift, y_shift) in enumerate(product(dx, dy)): i, j = np.unravel_index(n, (len(dx), len(dy))) if x_shift == 0 and y_shift == 0: self._scf_surface[j, i] = 1. if x_shift == 0: tmp = self.data else: if float(x_shift).is_integer(): shift_func = pixel_shift else: shift_func = fourier_shift tmp = shift_func(self.data, x_shift, axis=1) if y_shift != 0: if float(y_shift).is_integer(): shift_func = pixel_shift else: shift_func = fourier_shift tmp = shift_func(tmp, y_shift, axis=2) if boundary is "cut": # Always round up to the nearest integer. x_shift = np.ceil(x_shift).astype(int) y_shift = np.ceil(y_shift).astype(int) if x_shift < 0: x_slice_data = slice(None, tmp.shape[1] + x_shift) x_slice_tmp = slice(-x_shift, None) else: x_slice_data = slice(x_shift, None) x_slice_tmp = slice(None, tmp.shape[1] - x_shift) if y_shift < 0: y_slice_data = slice(None, tmp.shape[2] + y_shift) y_slice_tmp = slice(-y_shift, None) else: y_slice_data = slice(y_shift, None) y_slice_tmp = slice(None, tmp.shape[2] - y_shift) data_slice = (slice(None), x_slice_data, y_slice_data) tmp_slice = (slice(None), x_slice_tmp, y_slice_tmp) elif boundary is "continuous": data_slice = (slice(None),) * 3 tmp_slice = (slice(None),) * 3 values = \ np.nansum(((self.data[data_slice] - tmp[tmp_slice]) ** 2), axis=0) / \ (np.nansum(self.data[data_slice] ** 2, axis=0) + np.nansum(tmp[tmp_slice] ** 2, axis=0)) scf_value = 1. - \ np.sqrt(np.nansum(values) / np.sum(np.isfinite(values))) if scf_value > 1: raise ValueError("Cannot have a correlation above 1. Check " "your input data. Contact the TurbuStat " "authors if the problem persists.") self._scf_surface[j, i] = scf_value if show_progress: bar.update(n + 1)
column_flat, utline303, utline321, unoise)): if tcube[z,y,x] == 0: logh2column = np.log10(col)+22 mf.set_constraints(ratio303321=rat, eratio303321=erat, #ratio321322=ratio2, eratio321322=eratio2, logh2column=logh2column, elogh2column=elogh2column, logabundance=logabundance, elogabundance=elogabundance, taline303=ta303.value, etaline303=err, taline321=ta321.value, etaline321=err, linewidth=linewidth) row_data = mf.get_parconstraints() tcube[z,y,x] = row_data['temperature_chi2'] row_data['ratio303321'] = rat row_data['eratio303321'] = erat if ii % 100 == 0 or ii < 50: log.info("T: [{tmin1sig_chi2:7.2f},{temperature_chi2:7.2f},{tmax1sig_chi2:7.2f}] R={ratio303321:6.2f}+/-{eratio303321:6.2f}".format(**row_data)) else: pb.update(ii) tcube.flush() else: pb.update(ii) tcube[tcube==0] = np.nan tCube = SpectralCube(tcube, cube303.wcs, mask=BooleanArrayMask(np.isfinite(tcube), wcs=cube303.wcs)) tCube.write(hpath('chi2_temperature_cube.fits'), overwrite=True) print()
def dense_scatter(x, y, ax=None, zorder=None, label=None, marker='o', markersize=1., edgewidth=0., c='w', edgecolor='k', color_smoothing_box=None, xscale='log', yscale='log', show_progress=False, **kwargs): """ Make scatter plots that handle overlapping data points better. This function attempts to make a dense scatter plot prettier by: + merging the 'marker edges' of overlapping data points; + running a median-filter to homogenize color for 'neighbouring' data points (size of the smoothing box specified by the user). Parameters ---------- x, y : array_like x & y coordinates of the data points ax : `~matplotlib.axes.Axes` object, optional The Axes object in which to draw the scatter plot. zorder : float, optional label : string, optional Text label to use in the legend (ignored if facecolor is not a scalar) marker : marker style Default: 'o' markersize : float, optional Default: 1. edgewidth : float, optional Default: 0. c : color or array-like, optional Default: 'w' edgecolor : color, optional Default: 'k' color_smoothing_box : None or 2-tuple, optional If None, then no color smoothing will be performed. If a 2-tuple, then this parameter specifies the full width of the color smoothing box along X and Y direction. xscale : {'log', 'linear'}, optional X axis scale type (default: 'log') yscale : {'log', 'linear'}, optional Y axis scale type (default: 'log') show_progress : bool, optional Whether to show the progress bar for color smoothing. **kwargs Keywords to be passed to `~matplotlib.pyplot.scatter` Returns ------- ax : `~matplotlib.axes.Axes` object The Axes object in which contours are plotted. """ if ax is None: ax = plt.gca() ax.set_xscale(xscale) ax.set_yscale(yscale) if (color_smoothing_box is not None) and (np.size(c) > 1): if xscale == 'log': x_ = np.log10(x) elif xscale == 'linear': x_ = x else: raise ValueError("xscale={} not supported yet".format(xscale)) if yscale == 'log': y_ = np.log10(y) elif yscale == 'linear': y_ = y else: raise ValueError("yscale={} not supported yet".format(yscale)) newc = [] if show_progress: bar = ProgressBar(range(len(x))) else: bar = None for (x0_, y0_) in zip(x_, y_): newc.append( np.nanmedian(c[(x_ > x0_ - color_smoothing_box[0] / 2) & (x_ < x0_ + color_smoothing_box[0] / 2) & (y_ > y0_ - color_smoothing_box[1] / 2) & (y_ < y0_ + color_smoothing_box[1] / 2)])) if show_progress: bar.update() else: newc = c if edgewidth == 0: ax.scatter(x, y, marker=marker, c=newc, s=markersize**2, linewidths=0, zorder=zorder, **kwargs) else: ax.scatter(x, y, marker=marker, c=edgecolor, s=(markersize + edgewidth)**2, linewidths=0, zorder=zorder, **kwargs) ax.scatter(x, y, marker=marker, c=newc, s=(markersize - edgewidth)**2, linewidths=0, zorder=zorder, **kwargs) if label is not None: if np.size(c) > 1: print("Unable to add legend entry: `c` is not a scalar") else: ax.plot([], [], marker=marker, mfc=c, mec=edgecolor, ms=markersize, mew=edgewidth, ls='', label=label) return ax
def measure_dendrogram_properties(dend=None, cube303=cube303, cube321=cube321, cube13co=cube13co, cube18co=cube18co, noise_cube=noise_cube, sncube=sncube, suffix="", last_index=None, plot_some=True, line='303', write=True): assert (cube321.shape == cube303.shape == noise_cube.shape == cube13co.shape == cube18co.shape == sncube.shape) assert sncube.wcs is cube303.wcs is sncube.mask._wcs metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = 7.2 * u.arcsec metadata['beam_major'] = 30 * u.arcsec metadata['beam_minor'] = 30 * u.arcsec metadata['wavelength'] = 218.22219 * u.GHz metadata['velocity_scale'] = u.km / u.s metadata['wcs'] = cube303.wcs keys = [ 'density_chi2', 'expected_density', 'dmin1sig_chi2', 'dmax1sig_chi2', 'column_chi2', 'expected_column', 'cmin1sig_chi2', 'cmax1sig_chi2', 'temperature_chi2', 'expected_temperature', 'tmin1sig_chi2', 'tmax1sig_chi2', 'eratio321303', 'ratio321303', 'logh2column', 'elogh2column', 'logabundance', 'elogabundance', ] obs_keys = [ 'Stot303', 'Smin303', 'Smax303', 'Stot321', 'Smean303', 'Smean321', 'npix', 'e303', 'e321', 'r321303', 'er321303', '13cosum', 'c18osum', '13comean', 'c18omean', 's_ntotal', 'index', 'is_leaf', 'parent', 'root', 'lon', 'lat', 'vcen', 'higaldusttem', 'reff', 'dustmass', 'dustmindens', 'bad', #'tkin_turb', ] columns = {k: [] for k in (keys + obs_keys)} log.debug("Initializing dendrogram temperature fitting loop") # FORCE wcs to match # (technically should reproject here) cube13co._wcs = cube18co._wcs = cube303.wcs cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs if line == '303': maincube = cube303 elif line == '321': maincube = cube321 else: raise ValueError("Unrecognized line: {0}".format(line)) # Prepare an array to hold the fitted temperatures tcubedata = np.empty(maincube.shape, dtype='float32') tcubedata[:] = np.nan tcubeleafdata = np.empty(maincube.shape, dtype='float32') tcubeleafdata[:] = np.nan nbad = 0 catalog = ppv_catalog(dend, metadata) pb = ProgressBar(len(catalog)) for ii, row in enumerate(catalog): structure = dend[row['_idx']] assert structure.idx == row['_idx'] == ii dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs) dend_inds = structure.indices() view = ( slice(dend_inds[0].min(), dend_inds[0].max() + 1), slice(dend_inds[1].min(), dend_inds[1].max() + 1), slice(dend_inds[2].min(), dend_inds[2].max() + 1), ) #view2 = cube303.subcube_slices_from_mask(dend_obj_mask) submask = dend_obj_mask[view] #assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include()) sn = sncube[view].with_mask(submask) sntot = sn.sum().value #np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0) c303 = cube303[view].with_mask(submask) c321 = cube321[view].with_mask(submask) co13sum = cube13co[view].with_mask(submask).sum().value co18sum = cube18co[view].with_mask(submask).sum().value if hasattr(co13sum, '__len__'): raise TypeError( ".sum() applied to an array has yielded a non scalar.") npix = submask.include().sum() assert npix == structure.get_npix() Stot303 = c303.sum().value if np.isnan(Stot303): raise ValueError("NaN in cube. This can't happen: the data from " "which the dendrogram was derived can't have " "NaN pixels.") Smax303 = c303.max().value Smin303 = c303.min().value Stot321 = c321.sum().value if npix == 0: raise ValueError("npix=0. This is impossible.") Smean303 = Stot303 / npix if Stot303 <= 0 and line == '303': raise ValueError( "The 303 flux is <=0. This isn't possible because " "the dendrogram was derived from the 303 data with a " "non-zero threshold.") elif Stot303 <= 0 and line == '321': Stot303 = 0 Smean303 = 0 elif Stot321 <= 0 and line == '321': raise ValueError( "The 321 flux is <=0. This isn't possible because " "the dendrogram was derived from the 321 data with a " "non-zero threshold.") if np.isnan(Stot321): raise ValueError("NaN in 321 line") Smean321 = Stot321 / npix #error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5 var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2) error = var**0.5 if np.isnan(error): raise ValueError("error is nan: this is impossible by definition.") if line == '321' and Stot303 == 0: r321303 = np.nan er321303 = np.nan elif Stot321 < 0: r321303 = error / Smean303 er321303 = (r321303**2 * (var / Smean303**2 + 1))**0.5 else: r321303 = Stot321 / Stot303 er321303 = (r321303**2 * (var / Smean303**2 + var / Smean321**2))**0.5 for c in columns: assert len(columns[c]) == ii columns['index'].append(row['_idx']) columns['s_ntotal'].append(sntot) columns['Stot303'].append(Stot303) columns['Smax303'].append(Smax303) columns['Smin303'].append(Smin303) columns['Stot321'].append(Stot321) columns['Smean303'].append(Smean303) columns['Smean321'].append(Smean321) columns['npix'].append(npix) columns['e303'].append(error) columns['e321'].append(error) columns['r321303'].append(r321303) columns['er321303'].append(er321303) columns['13cosum'].append(co13sum) columns['c18osum'].append(co18sum) columns['13comean'].append(co13sum / npix) columns['c18omean'].append(co18sum / npix) columns['is_leaf'].append(structure.is_leaf) columns['parent'].append( structure.parent.idx if structure.parent else -1) columns['root'].append(get_root(structure).idx) s_main = maincube._data[dend_inds] x, y, z = maincube.world[dend_inds] lon = ((z.value - (360 * (z.value > 180))) * s_main).sum() / s_main.sum() lat = (y * s_main).sum() / s_main.sum() vel = (x * s_main).sum() / s_main.sum() columns['lon'].append(lon) columns['lat'].append(lat.value) columns['vcen'].append(vel.value) mask2d = dend_obj_mask.include().max(axis=0)[view[1:]] logh2column = np.log10( np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22) if np.isnan(logh2column): log.info("Source #{0} has NaNs".format(ii)) logh2column = 24 elogh2column = elogabundance columns['higaldusttem'].append( np.nanmean(dusttem_regridded.data[view[1:]][mask2d])) r_arcsec = row['radius'] * u.arcsec reff = (r_arcsec * (8.5 * u.kpc)).to(u.pc, u.dimensionless_angles()) mass = ((10**logh2column * u.cm**-2) * np.pi * reff**2 * 2.8 * constants.m_p).to(u.M_sun) density = (mass / (4 / 3. * np.pi * reff**3) / constants.m_p / 2.8).to( u.cm**-3) columns['reff'].append(reff.value) columns['dustmass'].append(mass.value) columns['dustmindens'].append(density.value) mindens = np.log10(density.value) if mindens < 3: mindens = 3 if (r321303 < 0 or np.isnan(r321303)) and line != '321': raise ValueError("Ratio <0: This can't happen any more because " "if either num/denom is <0, an exception is " "raised earlier") #for k in columns: # if k not in obs_keys: # columns[k].append(np.nan) elif (r321303 < 0 or np.isnan(r321303)) and line == '321': for k in keys: columns[k].append(np.nan) else: # Replace negatives for fitting if Smean321 <= 0: Smean321 = error mf.set_constraints( ratio321303=r321303, eratio321303=er321303, #ratio321322=ratio2, eratio321322=eratio2, logh2column=logh2column, elogh2column=elogh2column, logabundance=logabundance, elogabundance=elogabundance, taline303=Smean303, etaline303=error, taline321=Smean321, etaline321=error, mindens=mindens, linewidth=10) row_data = mf.get_parconstraints() row_data['ratio321303'] = r321303 row_data['eratio321303'] = er321303 for k in row_data: columns[k].append(row_data[k]) # Exclude bad velocities from cubes if row['v_cen'] < -80e3 or row['v_cen'] > 180e3: # Skip: there is no real structure down here nbad += 1 is_bad = True else: is_bad = False tcubedata[ dend_obj_mask.include()] = row_data['expected_temperature'] if structure.is_leaf: tcubeleafdata[dend_obj_mask.include( )] = row_data['expected_temperature'] columns['bad'].append(is_bad) width = row['v_rms'] * u.km / u.s lengthscale = reff #REMOVED in favor of despotic version done in dendrograms.py # we use the analytic version here; the despotic version is # computed elsewhere (with appropriate gcor factors) #columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3, # width, # lengthscale, # width/lengthscale, # columns['higaldusttem'][-1]*u.K, # crir=0./u.s)) if len(set(len(c) for k, c in columns.items())) != 1: print("Columns are different lengths. This is not allowed.") import ipdb ipdb.set_trace() for c in columns: assert len(columns[c]) == ii + 1 if plot_some and not is_bad and (ii - nbad % 100 == 0 or ii - nbad < 50): try: log.info( "T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]" " R={ratio321303:8.4f}+/-{eratio321303:8.4f}" " Smean303={Smean303:8.4f} +/- {e303:8.4f}" " Stot303={Stot303:8.2e} npix={npix:6d}".format( Smean303=Smean303, Stot303=Stot303, npix=npix, e303=error, **row_data)) pl.figure(1) pl.clf() mf.denstemplot() pl.savefig( fpath("dendrotem/diagnostics/{0}_{1}.png".format( suffix, ii))) pl.figure(2).clf() mf.parplot1d_all(levels=[0.68268949213708585]) pl.savefig( fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format( suffix, ii))) pl.draw() pl.show() except Exception as ex: print ex pass else: pb.update(ii + 1) if last_index is not None and ii >= last_index: break if last_index is not None: catalog = catalog[:last_index + 1] for k in columns: if k not in catalog.keys(): catalog.add_column(table.Column(name=k, data=columns[k])) for mid, lo, hi, letter in (('expected_temperature', 'tmin1sig_chi2', 'tmax1sig_chi2', 't'), ('expected_density', 'dmin1sig_chi2', 'dmax1sig_chi2', 'd'), ('expected_column', 'cmin1sig_chi2', 'cmax1sig_chi2', 'c')): catalog.add_column( table.Column(name='elo_' + letter, data=catalog[mid] - catalog[lo])) catalog.add_column( table.Column(name='ehi_' + letter, data=catalog[hi] - catalog[mid])) if write: catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)), format='ascii.ipac') # Note that there are overlaps in the catalog, which means that ORDER MATTERS # in the above loop. I haven't yet checked whether large scale overwrites # small or vice-versa; it may be that both views of the data are interesting. tcube = SpectralCube( data=tcubedata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit': 'K'}, header=cube303.header, ) tcubeleaf = SpectralCube( data=tcubeleafdata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit': 'K'}, header=cube303.header, ) if write: log.info("Writing TemperatureCube") outpath = 'TemperatureCube_DendrogramObjects{0}.fits' tcube.write(hpath(outpath.format(suffix)), overwrite=True) outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits' tcubeleaf.write(hpath(outpath_leaf.format(suffix)), overwrite=True) return catalog, tcube
def gaussfit_catalog( fitsfile, region_list, radius=1.0 * u.arcsec, max_radius_in_beams=2, max_offset_in_beams=1, background_estimator=np.nanmedian, noise_estimator=lambda x: mad_std(x, ignore_nan=True), savepath=None, prefix="", covariance='param_cov', raise_for_failure=False, ): """ Given a FITS filename and a list of regions, fit a gaussian to each region with an input guess based on the beam size. Parameters ---------- fitsfile : str Name of the FITS file region_list : list List of regions (see https://github.com/astropy/regions/) radius : angular size The radius of the region around the region center to extract and include in the fit max_radius_in_beams : float The maximum allowed source radius in units of beam major axis (this is a limit passed to the fitter) max_offset_in_beams : float The maximum allowed offset of the source center from the guessed position background_estimator : function A function to apply to the background pixels (those not within 1 beam HWHM of the center) to estimate the background level. The background will be subtracted before fitting. noise_estimator : function Function to apply to the whole data set to determine the noise level and therefore the appropriate per-pixel weight to get the correct normalization for the covariance matrix. savepath : str or None If specified, plots will be made and saved to this directory using the source name from the region metadata prefix : str The prefix to append to saved source names covariance : 'param_cov' or 'cov_x' Which covariance matrix should be used to estimate the parameter errors? ``param_cov`` uses the diagonal of the reduced-chi^2-scaled covariance matrix to compute the parameter errors, while ``cov_x`` uses the unscaled errors. See http://arxiv.org/abs/1009.2755 for a description, and criticism, of using the scaled covariance. raise_for_failure : bool If the fit was not successful, raise an exception """ # need central coordinates of each object coords = coordinates.SkyCoord([reg.center for reg in region_list]) fh = fits.open(fitsfile) data = fh[0].data.squeeze() header = fh[0].header datawcs = wcs.WCS(header).celestial beam = Beam.from_fits_header(header) pixscale = wcs.utils.proj_plane_pixel_area(datawcs)**0.5 * u.deg bmmin_px = (beam.minor.to(u.deg) / pixscale).decompose() bmmaj_px = (beam.major.to(u.deg) / pixscale).decompose() noise = noise_estimator(data) log.info("Noise estimate is {0} for file {1}".format(noise, fitsfile)) fit_data = {} pb = ProgressBar(len(region_list)) for ii, reg in enumerate(region_list): phot_reg = regions.CircleSkyRegion(center=reg.center, radius=radius) pixreg = phot_reg.to_pixel(datawcs) mask = pixreg.to_mask() mask_cutout = mask.cutout(data) if mask_cutout is None: log.warning( "Skipping region {0} because it failed to produce a cutout.". format(reg)) continue cutout = mask_cutout * mask.data cutout_mask = mask.data.astype('bool') smaller_phot_reg = regions.CircleSkyRegion(center=reg.center, radius=beam.major / 2.) #FWHM->HWHM smaller_pixreg = smaller_phot_reg.to_pixel(datawcs) smaller_mask = smaller_pixreg.to_mask() smaller_cutout = smaller_mask.cutout(data) * smaller_mask.data # mask out (as zeros) neighboring sources within the fitting area nearby_matches = phot_reg.contains(coords, datawcs) if any(nearby_matches): inds = np.where(nearby_matches)[0].tolist() inds.remove(ii) for ind in inds: maskoutreg = regions.EllipseSkyRegion( center=region_list[ind].center, width=beam.major, height=beam.minor, angle=beam.pa + 90 * u.deg, ) mpixreg = maskoutreg.to_pixel(datawcs) mmask = mpixreg.to_mask() view, mview = slice_bbox_from_bbox(mask.bbox, mmask.bbox) cutout_mask[view] &= ~mmask.data.astype('bool')[mview] cutout = cutout * cutout_mask background_mask = cutout_mask.copy().astype('bool') background_mask[sub_bbox_slice( mask.bbox, smaller_mask.bbox)] &= ~smaller_mask.data.astype('bool') background = background_estimator(cutout[background_mask]) sz = cutout.shape[0] mx = np.nanmax(smaller_cutout) ampguess = mx - background p_init = models.Gaussian2D( amplitude=ampguess, x_mean=sz / 2, y_mean=sz / 2, x_stddev=bmmaj_px / STDDEV_TO_FWHM, y_stddev=bmmin_px / STDDEV_TO_FWHM, theta=beam.pa, bounds={ 'x_stddev': (bmmin_px / STDDEV_TO_FWHM * 0.75, bmmaj_px * max_radius_in_beams / STDDEV_TO_FWHM), 'y_stddev': (bmmin_px / STDDEV_TO_FWHM * 0.75, bmmaj_px * max_radius_in_beams / STDDEV_TO_FWHM), 'x_mean': (sz / 2 - max_offset_in_beams * bmmaj_px / STDDEV_TO_FWHM, sz / 2 + max_offset_in_beams * bmmaj_px / STDDEV_TO_FWHM), 'y_mean': (sz / 2 - max_offset_in_beams * bmmaj_px / STDDEV_TO_FWHM, sz / 2 + max_offset_in_beams * bmmaj_px / STDDEV_TO_FWHM), 'amplitude': (ampguess * 0.9, ampguess * 1.1) }) imtofit = np.nan_to_num((cutout - background) * mask.data) result, fit_info, chi2, fitter = gaussfit_image( image=imtofit, gaussian=p_init, weights=1 / noise**2, plot=savepath is not None, ) if 'text' in reg.meta: sourcename = reg.meta['text'].strip('{}') elif 'label' in reg.meta: sourcename = reg.meta['label'].strip('{}') else: raise ValueError("Regions need to have names, either as 'text' or " "'label' entries.") if savepath is not None: with warnings.catch_warnings(): warnings.simplefilter('ignore', UserWarning) bmarr = beam.as_kernel(pixscale=pixscale, x_size=sz, y_size=sz).array assert bmarr.max() > 0 bm_ellipse = beam.ellipse_to_plot(sz / 2, sz / 2., pixscale) bm_ellipse.set_facecolor('none') bm_ellipse.set_edgecolor('r') pl.gca().add_patch(bm_ellipse) #pl.contour(bmarr, levels=[0.317*bmarr.max()], colors=['r']) pl.savefig(os.path.join(savepath, '{0}{1}.png'.format(prefix, sourcename)), bbox_inches='tight') if covariance not in fit_info or fit_info[covariance] is None: fit_info[covariance] = np.zeros([6, 6]) success = False else: success = True cx, cy = pixreg.bounding_box.ixmin + result.x_mean, pixreg.bounding_box.iymin + result.y_mean clon, clat = datawcs.wcs_pix2world(cx, cy, 0) major, minor = (result.x_stddev * STDDEV_TO_FWHM * pixscale.to(u.arcsec), result.y_stddev * STDDEV_TO_FWHM * pixscale.to(u.arcsec)) majind, minind = 3, 4 pa = (result.theta * u.rad).to(u.deg) if minor > major: major, minor = minor, major majind, minind = minind, majind pa += 90 * u.deg fitted_gaussian_as_beam = Beam(major=major, minor=minor, pa=pa) try: deconv_fit = fitted_gaussian_as_beam.deconvolve(beam) deconv_major, deconv_minor, deconv_pa = (deconv_fit.major, deconv_fit.minor, deconv_fit.pa) except ValueError: print("Could not deconvolve {0} from {1}".format( beam.__repr__(), fitted_gaussian_as_beam.__repr__())) deconv_major, deconv_minor, deconv_pa = np.nan, np.nan, np.nan fit_data[sourcename] = { 'amplitude': result.amplitude, 'center_x': float(clon) * u.deg, 'center_y': float(clat) * u.deg, 'fwhm_major': major, 'fwhm_minor': minor, 'pa': pa, 'deconv_fwhm_major': deconv_major, 'deconv_fwhm_minor': deconv_minor, 'deconv_pa': deconv_pa, 'chi2': chi2, 'chi2/n': chi2 / mask.data.sum(), 'e_amplitude': fit_info[covariance][0, 0]**0.5, 'e_center_x': fit_info[covariance][1, 1]**0.5 * pixscale, 'e_center_y': fit_info[covariance][2, 2]**0.5 * pixscale, 'e_fwhm_major': fit_info[covariance][majind, majind]**0.5 * STDDEV_TO_FWHM * pixscale.to(u.arcsec), 'e_fwhm_minor': fit_info[covariance][minind, minind]**0.5 * STDDEV_TO_FWHM * pixscale.to(u.arcsec), 'e_pa': fit_info[covariance][5, 5]**0.5 * u.deg, 'success': success, 'ampguess': ampguess, 'peak': mx, 'fit_info': fit_info, } if raise_for_failure and not success: raise ValueError("Fit failed.") pb.update(ii) signal.signal(signal.SIGINT, signal_handler) return fit_data
def download_timerange_log(start="2018-03-01", end=None, which="completed", nprocess=1, auth=None, show_progress=True, notebook=True, verbose=True): """ Storing and not return forced. See download_completed_log() for individual date downloading. """ if nprocess is None: nprocess = 1 elif nprocess < 1: raise ValueError("nprocess must 1 or higher (None means 1)") if auth is not None: print("updating skyvision authentification") io.set_account("skyvision", username=auth[0], password=auth[0], test=False) if which not in ["completed", "qa"]: raise ValueError(f"which can only be completed of qa, {which} given") dl_function = eval(f"download_{which}_log") # Dates to be downloaded. dates = get_daterange(start, end=None) if nprocess == 1: # Single processing if verbose: warnings.warn("No parallel downloading") _ = [ dl_function(date, auth=auth, store=True, returns=False) for date in dates ] else: # First download manually otherwise it could fail, unclear why _ = dl_function(dates[0], auth=auth, store=True, returns=False) # Multi processing import multiprocessing if show_progress: from astropy.utils.console import ProgressBar bar = ProgressBar(len(dates[1:]), ipython_widget=notebook) else: bar = None if verbose: warnings.warn( f"parallel downloading ; asking for {nprocess} processes") # Passing arguments with multiprocessing.Pool(nprocess) as p: # Da Loop for j, result in enumerate(p.imap(dl_function, dates[1:])): if bar is not None: bar.update(j) if bar is not None: bar.update(len(dates[1:]))
def combine_matches(table1, table2): """ Iterate through all stars in a stack of two tables, subtracting a test star's x and y from the rest of the catalog's x_cen and y_cen columns, sorting by x_cen, and testing distances starting at the top until the criterion is met.""" complete_colnames = set(table1.colnames + table2.colnames) stack = vstack([table1, table2]) stack = stack[sorted(list(complete_colnames))] print('Combining matches') numbefore = len(stack[np.where(stack['rejected'] == 0)]) pb = ProgressBar(numbefore) i = 0 while True: if i == len(stack) - 1: break if stack[i]['rejected'] == 1: i += 1 continue teststar = stack[i] diff_table = vstack([stack[:i], stack[i + 1:]])['_idx', 'x_cen', 'y_cen', 'position_angle'] diff_table['x_cen'] = np.abs(diff_table['x_cen'] - teststar['x_cen']) diff_table['y_cen'] = np.abs(diff_table['y_cen'] - teststar['y_cen']) diff_table.sort('x_cen') threshold = 1e-5 found_match = False dist_col = MaskedColumn(length=len(diff_table), name='distance', mask=True) for j in range( 10): # speed up computation by only going through 10 closest dist_col[j] = dist(diff_table[j]['x_cen'], diff_table[j]['y_cen']) if dist_col[j] <= threshold: found_match = True diff_table.add_column(dist_col) diff_table.sort('distance') if found_match: match_index = getrowindex(diff_table[0]['_idx'], diff_table[0]['position_angle'], stack) match = deepcopy(stack[match_index]) stack.remove_row(match_index) # Find the common bounding ellipse between the match and the test star new_x_cen = np.average([match['x_cen'], teststar['x_cen']]) new_y_cen = np.average([match['y_cen'], teststar['y_cen']]) # REPLACE WITH COMMON BOUNDING ELLIPSE new_major, new_minor, new_pa = commonbeam( match['major_fwhm'] * u.deg, match['minor_fwhm'] * u.deg, match['position_angle'] * u.deg, teststar['major_fwhm'] * u.deg, teststar['minor_fwhm'] * u.deg, teststar['position_angle'] * u.deg) # Save new info in test star's place stack[i]['x_cen'] = new_x_cen stack[i]['y_cen'] = new_y_cen stack[i]['major_fwhm'] = new_major.value stack[i]['minor_fwhm'] = new_minor.value stack[i]['position_angle'] = new_pa.value # Replace any masked data in the teststar row with available data from the match for k, masked in enumerate(stack.mask[i]): # get masked fields colname = stack.colnames[k] # get column name of masked fields if masked: # if masked: stack[i][colname] = match[ colname] # replace with data from the matched star i += 1 pb.update() for colname in stack.colnames: # iterate over columns if colname.split('_')[0] == 'detected': # if it's a detection column stack[ colname].fill_value = 0 # replace masked values with 0 (False) numafter = len(stack[np.where(stack['rejected'] == 0)]) print("\n{} matches combined".format(numbefore - numafter)) return stack
def test_files(filename, erasebad=True, nprocess=1, show_progress=True, notebook=False, redownload=False, **kwargs): """ Parameters ---------- filename: [fiulepath or list of] File(s) to be checked. erasebad: [bool] -optional- Do you want to remove from your local directory the corrupted files ? redownload: [bool] -optional- Shall corrupted file be automatically re downloaded ? (Only works for IRSA files ('/sci/','/raw/', '/ref/', '/cal/') nprocess: [int] -optional- Number of paralell processing show_progress: [bool] -optional- Do you want to show the progress bar ? notebook: [bool] Are you running from a notebook. Ignored if show_progress=False Returns ------- list of corrupted/bad files (might already be removed, see erasebad) """ if nprocess is None: nprocess = 1 elif nprocess < 1: raise ValueError("nprocess must 1 or higher (None means 1)") filename = np.atleast_1d(filename) if nprocess == 1: fileissue = [ f for f in filename if not _test_file_( f, erasebad=erasebad, redownload=redownload, **kwargs) ] else: import multiprocessing if show_progress: from astropy.utils.console import ProgressBar bar = ProgressBar(len(filename), ipython_widget=notebook) else: bar = None erasebad_ = [erasebad] * len(filename) fileissue = [] with multiprocessing.Pool(nprocess) as p: # Da Loop for j, isgood in enumerate( p.imap(_test_file_multiprocess_, zip(filename, erasebad_))): if bar is not None: bar.update(j) if not isgood: fileissue.append(filename[j]) if bar is not None: bar.update(len(filename)) if len(fileissue) > 0: warnings.warn("%d file failed" % len(fileissue)) if redownload: from .buildurl import _localsource_to_source_ to_download_urls, locations = np.asarray([ _localsource_to_source_(filename) for filename in fileissue ]).T source_to_dl = ["irsa"] for source in source_to_dl: source_dl = np.in1d(locations, [source]) print("Downloading %d files from %s" % (len(source_dl[source_dl]), source)) download_url(np.asarray(to_download_urls)[source_dl], np.asarray(fileissue)[source_dl], show_progress=show_progress, notebook=notebook, verbose=True, overwrite=True, nprocess=nprocess, cookies=get_cookie(*_load_id_(source)), **kwargs) for source_ in np.unique(locations): if source_ is not None and source_ not in source_to_dl: warnings.warn( "files from %s have not downloaded (not implemented)." % source_) return fileissue
def detect(infile, region, band, min_value=0.000325, min_delta=0.0005525, min_npix=7.5, plot=False, verbose=True): outfile = 'region{}_band{}_val{:.5g}_delt{:.5g}_pix{}'.format( region, band, min_value, min_delta, min_npix) contfile = fits.open(infile) # load in fits image da = contfile[0].data.squeeze() # get rid of extra axes print(da) mywcs = wcs.WCS( contfile[0].header ).celestial # set up world coordinate system, ditch extra dimensions beam = radio_beam.Beam.from_fits_header(contfile[0].header) d = Dendrogram.compute(da, min_value=min_value, min_delta=min_delta, min_npix=min_npix, wcs=mywcs, verbose=verbose) pixel_scale = np.abs( mywcs.pixel_scale_matrix.diagonal().prod())**0.5 * u.deg metadata = { 'data_unit': u.Jy / u.beam, 'spatial_scale': pixel_scale, 'beam_major': beam.major, 'beam_minor': beam.minor, 'wavelength': 9.298234612192E+10 * u.Hz, 'velocity_scale': u.km / u.s, 'wcs': mywcs, } cat = pp_catalog(d.leaves, metadata) # set up position-position catalog cat['_idx'] = range(len(cat)) # Use FWHM for ellipse dimensions instead of sigma cat['major_sigma'] = cat['major_sigma'] * np.sqrt(8 * np.log(2)) cat['minor_sigma'] = cat['minor_sigma'] * np.sqrt(8 * np.log(2)) cat.rename_column('major_sigma', 'major_fwhm') cat.rename_column('minor_sigma', 'minor_fwhm') cat.rename_column('flux', 'dend_flux_band{}'.format(band)) # Rename _idx to include the band number in the hundreds digit for i in range(len(cat)): cat['_idx'][i] = int('{}{:02d}'.format(band, cat['_idx'][i])) # Output the catalog and region files cat.write('./cat/cat_' + outfile + '.dat', format='ascii') savereg(cat, './reg/reg_' + outfile + '.reg') if plot: # create PDF plots of contour regions, if enabled ax = plt.gca() ax.cla() plt.imshow(da, cmap='gray_r', interpolation='none', origin='lower', vmax=0.01, vmin=-0.001) pltr = d.plotter() if verbose: print("Plotting contours to PDF...") pb = ProgressBar(len(d.leaves)) for struct in d.leaves: # iterate over each of the leaf structures pltr.plot_contour(ax, structure=struct, colors=['r'], linewidths=[0.9], zorder=5) if struct.parent: while struct.parent: struct = struct.parent pltr.plot_contour(ax, structure=struct, colors=[(0, 1, 0, 1)], linewidths=[0.5]) if verbose: pb.update() cntr = plt.gca().collections plt.setp([x for x in cntr if x.get_color()[0, 0] == 1], linewidth=0.25) plt.setp([x for x in cntr if x.get_color()[0, 1] == 1], linewidth=0.25) plt.savefig('./contour/contour_' + outfile + '.pdf') plt.axis((1125.4006254228616, 1670.3650637799306, 1291.6829155596627, 1871.8063499397681)) plt.setp([x for x in cntr if x.get_color()[0, 0] == 1], linewidth=0.75) # Red plt.setp([x for x in cntr if x.get_color()[0, 1] == 1], linewidth=0.5) # Green plt.savefig('./contour/contour_' + outfile + 'zoom.pdf')
def compute_dendro(self, show_progress=False, save_dendro=False, dendro_name=None, dendro_obj=None, periodic_bounds=False): ''' Compute the dendrogram and prune to the minimum deltas. ** min_deltas must be in ascending order! ** Parameters ---------- show_progress : optional, bool Enables the progress bar in astrodendro. save_dendro : optional, bool Saves the dendrogram in HDF5 format. **Requires pyHDF5** dendro_name : str, optional Save name when save_dendro is enabled. ".hdf5" appended automatically. dendro_obj : Dendrogram, optional Input a pre-computed dendrogram object. It is assumed that the dendrogram has already been computed! periodic_bounds : bool, optional Enable when the data is periodic in the spatial dimensions. ''' self._numfeatures = np.empty(self.min_deltas.shape, dtype=int) self._values = [] if dendro_obj is None: if periodic_bounds: # Find the spatial dimensions num_axes = self.data.ndim spat_axes = [] for i, axis_type in enumerate(self._wcs.get_axis_types()): if axis_type["coordinate_type"] == u"celestial": spat_axes.append(num_axes - i - 1) neighbours = periodic_neighbours(spat_axes) else: neighbours = None d = Dendrogram.compute(self.data, verbose=show_progress, min_delta=self.min_deltas[0], min_value=self.dendro_params["min_value"], min_npix=self.dendro_params["min_npix"], neighbours=neighbours) else: d = dendro_obj self._numfeatures[0] = len(d) self._values.append(np.array([struct.vmax for struct in d.all_structures])) if len(self.min_deltas) > 1: # Another progress bar for pruning steps if show_progress: print("Pruning steps.") bar = ProgressBar(len(self.min_deltas[1:])) for i, delta in enumerate(self.min_deltas[1:]): d.prune(min_delta=delta) self._numfeatures[i + 1] = len(d) self._values.append(np.array([struct.vmax for struct in d.all_structures])) if show_progress: bar.update(i + 1)
dens_slopes.append(dens_slope.slope) cube_hdu = make_ppv(velocity, density, vel_disp=np.std(velocity), T=T, threads=4, verbose=True, chan_width=dv_eff / 2., v_min=-60 * u.km / u.s, v_max=60 * u.km / u.s, max_chan=2000) filename = "fBM_density_{0:.2f}_velocity_{1:.2f}_rep_{2}_size_{3}.fits"\ .format(np.abs(dens), np.abs(vel), i, cube_size) cube_hdu.writeto(osjoin(out_dir, filename), overwrite=True) bar.update() filename = "fBM_3D_velocity_slopes_size_{0}.npy"\ .format(cube_size) np.save(osjoin(out_dir, filename), np.array(vel_slopes)) filename = "fBM_3D_density_slopes_size_{0}.npy"\ .format(cube_size) np.save(osjoin(out_dir, filename), np.array(dens_slopes))
logabundance=logabundance, elogabundance=elogabundance, taline303=ta303.value, etaline303=err, taline321=ta321.value, etaline321=err, linewidth=linewidth) row_data = mf.get_parconstraints() tcube[z, y, x] = row_data['temperature_chi2'] row_data['ratio303321'] = rat row_data['eratio303321'] = erat if ii % 100 == 0 or ii < 50: log.info( "T: [{tmin1sig_chi2:7.2f},{temperature_chi2:7.2f},{tmax1sig_chi2:7.2f}] R={ratio303321:6.2f}+/-{eratio303321:6.2f}" .format(**row_data)) else: pb.update(ii) tcube.flush() else: pb.update(ii) tcube[tcube == 0] = np.nan tCube = SpectralCube(tcube, cube303.wcs, mask=BooleanArrayMask(np.isfinite(tcube), wcs=cube303.wcs)) tCube.write(hpath('chi2_temperature_cube.fits'), overwrite=True) print()
def var_cov_cube(cube, mean_sub=False, progress_bar=True): ''' Compute the variance-covariance matrix of a data cube, with proper handling of NaNs. Parameters ---------- cube : numpy.ndarray PPV cube. Spectral dimension assumed to be 0th axis. mean_sub : bool, optional Subtract column means. progress_bar : bool, optional Show a progress bar, since this operation could be slow for large cubes. Returns ------- cov_matrix : numpy.ndarray Computed covariance matrix. ''' n_velchan = cube.shape[0] cov_matrix = np.zeros((n_velchan, n_velchan)) if progress_bar: bar = ProgressBar(n_velchan) for i, chan in enumerate(_iter_2D(cube)): # Set the nans to tiny values chan[np.isnan(chan)] = np.finfo(chan.dtype).eps norm_chan = chan if mean_sub: norm_chan -= np.nanmean(chan) for j, chan2 in enumerate(_iter_2D(cube[:i + 1, :, :])): norm_chan2 = chan2 if mean_sub: norm_chan2 -= np.nanmean(chan2) divisor = np.sum(np.isfinite(norm_chan * norm_chan2)) # Apply Bessel's correction when mean subtracting if mean_sub: divisor -= 1.0 cov_matrix[i, j] = \ np.nansum(norm_chan * norm_chan2) / divisor # Variances # Divided in half to account for doubling in line below var_divis = np.sum(np.isfinite(norm_chan)) if mean_sub: var_divis -= 1.0 cov_matrix[i, i] = 0.5 * \ np.nansum(norm_chan * norm_chan) / var_divis if progress_bar: bar.update(i + 1) cov_matrix = cov_matrix + cov_matrix.T return np.nan_to_num(cov_matrix)
def var_cov_cube(cube, mean_sub=False, progress_bar=True): ''' Compute the variance-covariance matrix of a data cube, with proper handling of NaNs. Parameters ---------- cube : numpy.ndarray PPV cube. Spectral dimension assumed to be 0th axis. mean_sub : bool, optional Subtract column means. progress_bar : bool, optional Show a progress bar, since this operation could be slow for large cubes. Returns ------- cov_matrix : numpy.ndarray Computed covariance matrix. ''' n_velchan = cube.shape[0] cov_matrix = np.zeros((n_velchan, n_velchan)) if progress_bar: bar = ProgressBar(n_velchan) for i, chan in enumerate(_iter_2D(cube)): norm_chan = chan if mean_sub: norm_chan -= np.nanmean(chan) for j, chan2 in enumerate(_iter_2D(cube[:i + 1, :, :])): norm_chan2 = chan2 if mean_sub: norm_chan2 -= np.nanmean(chan2) divisor = np.sum(np.isfinite(norm_chan * norm_chan2)) # Apply Bessel's correction when mean subtracting if mean_sub: divisor -= 1.0 cov_matrix[i, j] = \ np.nansum(norm_chan * norm_chan2) / divisor # Variances # Divided in half to account for doubling in line below var_divis = np.sum(np.isfinite(norm_chan)) if mean_sub: var_divis -= 1.0 cov_matrix[i, i] = 0.5 * \ np.nansum(norm_chan * norm_chan) / var_divis if progress_bar: bar.update(i + 1) cov_matrix = cov_matrix + cov_matrix.T return np.nan_to_num(cov_matrix)
def add_data_to_cube(cubefilename, data=None, filename=None, fileheader=None, flatheader='header.txt', cubeheader='cubeheader.txt', nhits=None, smoothto=1, baselineorder=5, velocityrange=None, excludefitrange=None, noisecut=np.inf, do_runscript=False, linefreq=None, allow_smooth=True, data_iterator=data_iterator, coord_iterator=coord_iterator, velo_iterator=velo_iterator, progressbar=False, coordsys='galactic', datalength=None, velocity_offset=0.0, negative_mean_cut=None, add_with_kernel=False, kernel_fwhm=None, fsw=False, kernel_function=Gaussian2DKernel, diagnostic_plot_name=None, chmod=False, continuum_prefix=None, debug_breakpoint=False, default_unit=u.km/u.s, make_continuum=True, weightspec=None, varweight=False): """ Given a .fits file that contains a binary table of spectra (e.g., as you would get from the GBT mapping "pipeline" or the reduce_map.pro aoidl file provided by Adam Ginsburg), adds each spectrum into the cubefile. velocity_offset : 0.0 Amount to add to the velocity vector before adding it to the cube (useful for FSW observations) weightspec : np.ndarray A spectrum with the same size as the input arrays but containing the relative weights of the data """ #if not default_unit.is_equivalent(u.km/u.s): # raise TypeError("Default unit is not a velocity equivalent.") if type(nhits) is str: log.debug("Loading nhits from %s" % nhits) nhits = pyfits.getdata(nhits) elif type(nhits) is not np.ndarray: raise TypeError("nhits must be a .fits file or an ndarray, but it is ",type(nhits)) naxis2,naxis1 = nhits.shape if velocity_offset and not fsw: raise ValueError("Using a velocity offset, but obs type is not " "frequency switched; this is almost certainly wrong, " "but if there's a case for it I'll remove this.") if not hasattr(velocity_offset,'unit'): velocity_offset = velocity_offset*default_unit contimage = np.zeros_like(nhits) nhits_once = np.zeros_like(nhits) log.debug("Loading data cube {0}".format(cubefilename)) t0 = time.time() # rescale image to weight by number of observations image = pyfits.getdata(cubefilename)*nhits log.debug(" ".join(("nhits statistics: mean, std, nzeros, size",str(nhits.mean()),str(nhits.std()),str(np.sum(nhits==0)), str(nhits.size)))) log.debug(" ".join(("Image statistics: mean, std, nzeros, size",str(image.mean()),str(image.std()),str(np.sum(image==0)), str(image.size), str(np.sum(np.isnan(image)))))) log.debug(" ".join(("nhits shape: ",str(nhits.shape)))) # default is to set empty pixels to NAN; have to set them # back to zero image[image!=image] = 0.0 header = pyfits.getheader(cubefilename) # debug print "Cube shape: ",image.shape," naxis3: ",header.get('NAXIS3')," nhits shape: ",nhits.shape log.debug("".join(("Image statistics: mean, std, nzeros, size",str(image.mean()),str(image.std()),str(np.sum(image==0)), str(image.size)))) flathead = get_header(flatheader) naxis3 = image.shape[0] wcs = pywcs.WCS(flathead) cwcs = pywcs.WCS(header) vwcs = cwcs.sub([pywcs.WCSSUB_SPECTRAL]) vunit = u.Unit(vwcs.wcs.cunit[vwcs.wcs.spec]) cubevelo = vwcs.wcs_pix2world(np.arange(naxis3),0)[0] * vunit cd3 = vwcs.wcs.cdelt[vwcs.wcs.spec] * vunit if not vunit.is_equivalent(default_unit): raise ValueError("The units of the cube and the velocity axis are " "possibly not equivalent. Change default_unit to " "the appropriate unit (probably {0})".format(vunit)) if add_with_kernel: if wcs.wcs.has_cd(): cd = np.abs(wcs.wcs.cd[1,1]) else: cd = np.abs(wcs.wcs.cdelt[1]) # Alternative implementation; may not work for .cd? #cd = np.abs(np.prod((wcs.wcs.get_cdelt() * wcs.wcs.get_pc().diagonal())))**0.5 if velocityrange is not None: if hasattr(velocityrange, 'unit'): v1,v4 = velocityrange else: v1,v4 = velocityrange * default_unit ind1 = np.argmin(np.abs(np.floor(v1-cubevelo))) ind2 = np.argmin(np.abs(np.ceil(v4-cubevelo)))+1 # stupid hack. REALLY stupid hack. Don't crop. if np.abs(ind2-image.shape[0]) < 5: ind2 = image.shape[0] if np.abs(ind1) < 5: ind1 = 0 #print "Velo match for v1,v4 = %f,%f: %f,%f" % (v1,v4,cubevelo[ind1],cubevelo[ind2]) # print "Updating CRPIX3 from %i to %i. Cropping to indices %i,%i" % (header.get('CRPIX3'),header.get('CRPIX3')-ind1,ind1,ind2) # I think this could be disastrous: cubevelo is already set, but now we're changing how it's set in the header! # I don't think there's any reason to have this in the first place # header.set('CRPIX3',header.get('CRPIX3')-ind1) # reset v1,v4 to the points we just selected v1 = cubevelo[ind1] v4 = cubevelo[ind2-1] else: ind1=0 ind2 = image.shape[0] v1,v4 = min(cubevelo),max(cubevelo) # debug print "Cube has %i v-axis pixels from %f to %f. Crop range is %f to %f" % (naxis3,cubevelo.min(),cubevelo.max(),v1,v4) #if abs(cdelt) < abs(cd3): # print "Spectra have CD=%0.2f, cube has CD=%0.2f. Will smooth & interpolate." % (cdelt,cd3) # Disable progressbar if debug-logging is enabled (they clash) if progressbar and 'ProgressBar' in globals() and log.level > 10: if datalength is None: pb = ProgressBar(len(data)) else: pb = ProgressBar(datalength) else: progressbar = False skipped = [] for spectrum,pos,velo in zip(data_iterator(data,fsw=fsw), coord_iterator(data,coordsys_out=coordsys), velo_iterator(data,linefreq=linefreq)): if log.level <= 10: t1 = time.time() if not hasattr(velo,'unit'): velo = velo * default_unit glon,glat = pos cdelt = velo[1]-velo[0] if cdelt < 0: # for interpolation, require increasing X axis spectrum = spectrum[::-1] velo = velo[::-1] if log.level < 5: log.debug("Reversed spectral axis... ") if (velo.max() < cubevelo.min() or velo.min() > cubevelo.max()): raise ValueError("Data out of range.") if progressbar and log.level > 10: pb.update() velo += velocity_offset if glon != 0 and glat != 0: x,y = wcs.wcs_world2pix(glon,glat,0) if np.isnan(x) or np.isnan(y): log.warn("".join(("Skipping NaN point {0}, {1} ...".format(glon,glat)))) continue if log.level < 10: log.debug("".join(("At point {0},{1} ...".format(glon,glat),))) if abs(cdelt) < abs(cd3) and allow_smooth: # need to smooth before interpolating to preserve signal kernwidth = abs(cd3/cdelt/2.35).decompose().value if kernwidth > 2 and kernwidth < 10: xr = kernwidth*5 npx = np.ceil(xr*2 + 1) elif kernwidth > 10: raise ValueError('Too much smoothing') else: xr = 5 npx = 11 #kernel = np.exp(-(np.linspace(-xr,xr,npx)**2)/(2.0*kernwidth**2)) #kernel /= kernel.sum() kernel = Gaussian1DKernel(stddev=kernwidth, x_size=npx) smspec = np.convolve(spectrum,kernel,mode='same') datavect = np.interp(cubevelo.to(default_unit).value, velo.to(default_unit).value, smspec) else: datavect = np.interp(cubevelo.to(default_unit).value, velo.to(default_unit).value, spectrum) OK = (datavect[ind1:ind2] == datavect[ind1:ind2]) if excludefitrange is None: include = OK else: # Exclude certain regions (e.g., the spectral lines) when computing the noise include = OK.copy() if not hasattr(excludefitrange,'unit'): excludefitrange = excludefitrange * default_unit # Convert velocities to indices exclude_inds = [np.argmin(np.abs(np.floor(v-cubevelo))) for v in excludefitrange] # Loop through exclude_inds pairwise for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]): # Do not include the excluded regions include[i1:i2] = False if include.sum() == 0: raise ValueError("All data excluded.") noiseestimate = datavect[ind1:ind2][include].std() contestimate = datavect[ind1:ind2][include].mean() if noiseestimate > noisecut: log.info("Skipped a data point at %f,%f in file %s because it had excessive noise %f" % (x,y,filename,noiseestimate)) skipped.append(True) continue elif negative_mean_cut is not None and contestimate < negative_mean_cut: log.info("Skipped a data point at %f,%f in file %s because it had negative continuum %f" % (x,y,filename,contestimate)) skipped.append(True) continue elif OK.sum() == 0: log.info("Skipped a data point at %f,%f in file %s because it had NANs" % (x,y,filename)) skipped.append(True) continue elif OK.sum()/float(abs(ind2-ind1)) < 0.5: log.info("Skipped a data point at %f,%f in file %s because it had %i NANs" % (x,y,filename,np.isnan(datavect[ind1:ind2]).sum())) skipped.append(True) continue if log.level < 10: log.debug("did not skip...") if varweight: weight = 1./noiseestimate**2 else: weight = 1. if weightspec is None: wspec = weight else: wspec = weight * weightspec if 0 < int(np.round(x)) < naxis1 and 0 < int(np.round(y)) < naxis2: if add_with_kernel: fwhm = np.sqrt(8*np.log(2)) kernel_size = kd = int(np.ceil(kernel_fwhm/fwhm/cd * 5)) if kernel_size < 5: kernel_size = kd = 5 if kernel_size % 2 == 0: kernel_size = kd = kernel_size+1 if kernel_size > 100: raise ValueError("Huge kernel - are you sure?") kernel_middle = mid = (kd-1)/2. xinds,yinds = (np.mgrid[:kd,:kd]-mid+np.array([np.round(x),np.round(y)])[:,None,None]).astype('int') # This kernel is NOT centered, and that's the bloody point. # (I made a very stupid error and used Gaussian2DKernel, # which is strictly centered, in a previous version) kernel2d = np.exp(-((xinds-x)**2+(yinds-y)**2)/(2*(kernel_fwhm/fwhm/cd)**2)) dim1 = ind2-ind1 vect_to_add = np.outer(datavect[ind1:ind2],kernel2d).reshape([dim1,kd,kd]) vect_to_add[True-OK] = 0 # need to slice out edges if yinds.max() >= naxis2 or yinds.min() < 0: yok = (yinds[0,:] < naxis2) & (yinds[0,:] >= 0) xinds,yinds = xinds[:,yok],yinds[:,yok] vect_to_add = vect_to_add[:,:,yok] kernel2d = kernel2d[:,yok] if xinds.max() >= naxis1 or xinds.min() < 0: xok = (xinds[:,0] < naxis1) & (xinds[:,0] >= 0) xinds,yinds = xinds[xok,:],yinds[xok,:] vect_to_add = vect_to_add[:,xok,:] kernel2d = kernel2d[xok,:] image[ind1:ind2,yinds,xinds] += vect_to_add*wspec # NaN spectral bins are not appropriately downweighted... but they shouldn't exist anyway... nhits[yinds,xinds] += kernel2d*weight contimage[yinds,xinds] += kernel2d * contestimate*weight nhits_once[yinds,xinds] += kernel2d*weight else: image[ind1:ind2,int(np.round(y)),int(np.round(x))][OK] += datavect[ind1:ind2][OK]*weight nhits[int(np.round(y)),int(np.round(x))] += weight contimage[int(np.round(y)),int(np.round(x))] += contestimate*weight nhits_once[int(np.round(y)),int(np.round(x))] += weight if log.level < 10: log.debug("Z-axis indices are %i,%i..." % (ind1,ind2,)) log.debug("Added a data point at %i,%i" % (int(np.round(x)),int(np.round(y)))) skipped.append(False) else: skipped.append(True) log.info("Skipped a data point at x,y=%f,%f " "lon,lat=%f,%f in file %s because " "it's out of the grid" % (x,y,glon,glat,filename)) if debug_breakpoint: import ipdb ipdb.set_trace() if log.level <= 10: dt = time.time() - t1 log.debug("Completed x,y={x:4.0f},{y:4.0f}" " ({x:6.2f},{y:6.2f}) in {dt:6.2g}s".format(x=float(x), y=float(y), dt=dt)) log.info("Completed 'add_data' loop for" " {0} in {1}s".format(cubefilename, time.time()-t0)) if data.dtype.names is not None: dname = 'DATA' if 'DATA' in data.dtype.names else 'SPECTRA' else: dname = slice(None) if excludefitrange is not None: # this block redefining "include" is used for diagnostics (optional) ind1a = np.argmin(np.abs(np.floor(v1-velo))) ind2a = np.argmin(np.abs(np.ceil(v4-velo)))+1 OK = (data[dname][0,:]==data[dname][0,:]) OK[:ind1a] = False OK[ind2a:] = False include = OK # Convert velocities to indices exclude_inds = [np.argmin(np.abs(np.floor(v-velo))) for v in excludefitrange] # Loop through exclude_inds pairwise for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]): # Do not include the excluded regions include[i1:i2] = False if include.sum() == 0: raise ValueError("All data excluded.") else: include = slice(None) if diagnostic_plot_name: from mpl_plot_templates import imdiagnostics pylab.clf() dd = data[dname][:,include] imdiagnostics(dd,axis=pylab.gca()) pylab.savefig(diagnostic_plot_name, bbox_inches='tight') # Save a copy with the bad stuff flagged out; this should tell whether flagging worked skipped = np.array(skipped,dtype='bool') dd[skipped,:] = -999 maskdata = np.ma.masked_equal(dd,-999) pylab.clf() imdiagnostics(maskdata, axis=pylab.gca()) dpn_pre,dpn_suf = os.path.splitext(diagnostic_plot_name) dpn_flagged = dpn_pre+"_flagged"+dpn_suf pylab.savefig(dpn_flagged, bbox_inches='tight') log.info("Saved diagnostic plot %s and %s" % (diagnostic_plot_name,dpn_flagged)) log.debug("nhits statistics: mean, std, nzeros, size {0} {1} {2} {3}".format(nhits.mean(),nhits.std(),np.sum(nhits==0), nhits.size)) log.debug("Image statistics: mean, std, nzeros, size {0} {1} {2} {3}".format(image.mean(),image.std(),np.sum(image==0), image.size)) imav = image/nhits if log.level <= 10: nnan = np.count_nonzero(np.isnan(imav)) log.debug("imav statistics: mean, std, nzeros, size, nnan, ngood: {0} {1} {2} {3} {4} {5}".format(imav.mean(),imav.std(),np.sum(imav==0), imav.size, nnan, imav.size-nnan)) log.debug("imav shape: {0}".format(imav.shape)) subcube = imav[ind1:ind2,:,:] if log.level <= 10: nnan = np.sum(np.isnan(subcube)) print("subcube statistics: mean, std, nzeros, size, nnan, ngood:",np.nansum(subcube)/subcube.size,np.std(subcube[subcube==subcube]),np.sum(subcube==0), subcube.size, nnan, subcube.size-nnan) print("subcube shape: ",subcube.shape) H = header.copy() if fileheader is not None: for k,v in fileheader.items(): if 'RESTFRQ' in k or 'RESTFREQ' in k: header.set(k,v) #if k[0] == 'C' and '1' in k and k[-1] != '1': # header.set(k.replace('1','3'), v) moreH = get_header(cubeheader) for k,v in H.items(): header.set(k,v) for k,v in moreH.items(): header.set(k,v) HDU = pyfits.PrimaryHDU(data=subcube,header=header) HDU.writeto(cubefilename,clobber=True,output_verify='fix') outpre = cubefilename.replace(".fits","") include = np.ones(imav.shape[0],dtype='bool') if excludefitrange is not None: # this block redifining "include" is used for continuum ind1a = np.argmin(np.abs(np.floor(v1-cubevelo))) ind2a = np.argmin(np.abs(np.ceil(v4-cubevelo)))+1 # Convert velocities to indices exclude_inds = [np.argmin(np.abs(np.floor(v-cubevelo))) for v in excludefitrange] # Loop through exclude_inds pairwise for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]): # Do not include the excluded regions include[i1:i2] = False if include.sum() == 0: raise ValueError("All data excluded.") HDU2 = pyfits.PrimaryHDU(data=nhits,header=flathead) HDU2.writeto(outpre+"_nhits.fits",clobber=True,output_verify='fix') #OKCube = (imav==imav) #contmap = np.nansum(imav[naxis3*0.1:naxis3*0.9,:,:],axis=0) / OKCube.sum(axis=0) if make_continuum: contmap = np.nansum(imav[include,:,:],axis=0) / include.sum() HDU2 = pyfits.PrimaryHDU(data=contmap,header=flathead) HDU2.writeto(outpre+"_continuum.fits",clobber=True,output_verify='fix') if continuum_prefix is not None: # Solo continuum image (just this obs set) HDU2.data = contimage / nhits_once HDU2.writeto(continuum_prefix+"_continuum.fits",clobber=True,output_verify='fix') HDU2.data = nhits_once HDU2.writeto(continuum_prefix+"_nhits.fits",clobber=True,output_verify='fix') log.info("Writing script file {0}".format(outpre+"_starlink.sh")) scriptfile = open(outpre+"_starlink.sh",'w') outpath,outfn = os.path.split(cubefilename) outpath,pre = os.path.split(outpre) print(("#!/bin/bash"), file=scriptfile) if outpath != '': print(('cd %s' % outpath), file=scriptfile) print(('. /star/etc/profile'), file=scriptfile) print(('kappa > /dev/null'), file=scriptfile) print(('convert > /dev/null'), file=scriptfile) print(('fits2ndf %s %s' % (outfn,outfn.replace(".fits",".sdf"))), file=scriptfile) if excludefitrange is not None: v2v3 = "" for v2,v3 in zip(excludefitrange[::2],excludefitrange[1::2]): v2v3 += "%0.2f %0.2f " % (v2.to(default_unit).value,v3.to(default_unit).value) print(('mfittrend %s ranges=\\\"%0.2f %s %0.2f\\\" order=%i axis=3 out=%s' % (outfn.replace(".fits",".sdf"),v1.to(default_unit).value,v2v3,v4.to(default_unit).value,baselineorder,outfn.replace(".fits","_baseline.sdf"))), file=scriptfile) else: print(('mfittrend %s ranges=\\\"%0.2f %0.2f\\\" order=%i axis=3 out=%s' % (outfn.replace(".fits",".sdf"),v1.to(default_unit).value,v4.to(default_unit).value,baselineorder,outfn.replace(".fits","_baseline.sdf"))), file=scriptfile) print(('sub %s %s %s' % (outfn.replace(".fits",".sdf"),outfn.replace(".fits","_baseline.sdf"),outfn.replace(".fits","_sub.sdf"))), file=scriptfile) print(('sqorst %s_sub mode=pixelscale axis=3 pixscale=%i out=%s_vrebin' % (pre,smoothto,pre)), file=scriptfile) print(('gausmooth %s_vrebin fwhm=1.0 axes=[1,2] out=%s_smooth' % (pre,pre)), file=scriptfile) print(('#collapse %s estimator=mean axis="VRAD" low=-400 high=500 out=%s_continuum' % (pre,pre)), file=scriptfile) print(('rm %s_sub.fits' % (pre)), file=scriptfile) print(('ndf2fits %s_sub %s_sub.fits' % (pre,pre)), file=scriptfile) print(('rm %s_smooth.fits' % (pre)), file=scriptfile) print(('ndf2fits %s_smooth %s_smooth.fits' % (pre,pre)), file=scriptfile) print(("# Fix STARLINK's failure to respect header keywords."), file=scriptfile) print(('sethead %s_smooth.fits RESTFRQ=`gethead RESTFRQ %s.fits`' % (pre,pre)), file=scriptfile) print(('rm %s_baseline.sdf' % (pre)), file=scriptfile) print(('rm %s_smooth.sdf' % (pre)), file=scriptfile) print(('rm %s_sub.sdf' % (pre)), file=scriptfile) print(('rm %s_vrebin.sdf' % (pre)), file=scriptfile) print(('rm %s.sdf' % (pre)), file=scriptfile) scriptfile.close() if chmod: scriptfilename = (outpre+"_starlink.sh").replace(" ","") #subprocess.call("chmod +x {0}".format(scriptfilename), shell=True) st = os.stat(scriptfilename) os.chmod(scriptfilename, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH | stat.S_IXUSR) if do_runscript: runscript(outpre) _fix_ms_kms_file(outpre+"_sub.fits") _fix_ms_kms_file(outpre+"_smooth.fits") if log.level <= 20: log.info("Completed {0} in {1}s".format(pre, time.time()-t0))
def quick_render_movie(self, outdir, size=256, nframes=30, camera_angle=(0, 0, 1), north_vector=(0, 0, 1), rot_vector=(1, 0, 0), colormap='doom', cmap_range='auto', transfer_function='auto', start_index=0, image_prefix="", output_filename='out.mp4', log_scale=False, rescale=True): """ Create a movie rotating the cube 360 degrees from PP -> PV -> PP -> PV -> PP Parameters ---------- outdir: str The output directory in which the individual image frames and the resulting output mp4 file should be stored size: int The size of the individual output frame in pixels (i.e., size=256 will result in a 256x256 image) nframes: int The number of frames in the resulting movie camera_angle: 3-tuple The initial angle of the camera north_vector: 3-tuple The vector of 'north' in the data cube. Default is coincident with the spectral axis rot_vector: 3-tuple The vector around which the camera will be rotated colormap: str A valid colormap. See `yt.show_colormaps` transfer_function: 'auto' or `yt.visualization.volume_rendering.TransferFunction` Either 'auto' to use the colormap specified, or a valid TransferFunction instance log_scale: bool Should the colormap be log scaled? rescale: bool If True, the images will be rescaled to have a common 95th percentile brightness, which can help reduce flickering from having a single bright pixel in some projections start_index : int The number of the first image to save image_prefix : str A string to prepend to the image name for each image that is output output_filename : str The movie file name to output. The suffix may affect the file type created. Defaults to 'out.mp4'. Will be placed in ``outdir`` Returns ------- """ if not ytOK: raise IOError( "yt could not be imported. Cube renderings are not possible.") scale = np.max(self.cube.shape) if not os.path.exists(outdir): os.makedirs(outdir) elif not os.path.isdir(outdir): raise OSError( "Output directory {0} exists and is not a directory.".format( outdir)) if cmap_range == 'auto': upper = self.cube.max().value lower = self.cube.std().value * 3 cmap_range = [lower, upper] if transfer_function == 'auto': tfh = self.auto_transfer_function(cmap_range, log=log_scale) tfh.tf.map_to_colormap(cmap_range[0], cmap_range[1], colormap=colormap) tf = tfh.tf else: tf = transfer_function center = self.dataset.domain_center cam = self.dataset.h.camera(center, camera_angle, scale, size, tf, north_vector=north_vector, fields='flux') im = cam.snapshot() images = [im] pb = ProgressBar(nframes) for ii, im in enumerate( cam.rotation(2 * np.pi, nframes, rot_vector=rot_vector)): images.append(im) im.write_png(os.path.join( outdir, "%s%04i.png" % (image_prefix, ii + start_index)), rescale=False) pb.update(ii + 1) log.info("Rendering complete in {0}s".format(time.time() - pb._start_time)) if rescale: _rescale_images(images, os.path.join(outdir, image_prefix)) pipe = _make_movie(outdir, prefix=image_prefix, filename=output_filename) return images
def convolve_model_dir(model_dir, filters, overwrite=False): """ Convolve all the model SEDs in a model directory Parameters ---------- model_dir : str The path to the model directory filters : list A list of :class:`~sedfitter.filter.Filter` objects to use for the convolution overwrite : bool, optional Whether to overwrite the output files """ for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') + glob.glob(model_dir + '/seds/*/*.fits.gz') + glob.glob(model_dir + '/seds/*.fits') + glob.glob(model_dir + '/seds/*/*.fits')) par_table = load_parameter_table(model_dir) if len(sed_files) == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(len(sed_files), model_dir)) # Find out apertures first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures # Set up convolved fluxes fluxes = [ConvolvedFluxes(model_names=np.zeros(len(sed_files), dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(len(filters))] # Set up list of binned filters binned_filters = [] binned_nu = None # Loop over SEDs b = ProgressBar(len(sed_files)) for im, sed_file in enumerate(sed_files): log.debug('Convolving {0}'.format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu') # Check if filters need to be re-binned try: assert binned_nu is not None np.testing.assert_array_almost_equal_nulp(s.nu.value, binned_nu.value, 100) except AssertionError: log.info('Rebinning filters') binned_filters = [f.rebin(s.nu) for f in filters] binned_nu = s.nu b.update() # Convolve for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = apertures fluxes[i].model_names[im] = s.name if n_ap == 1: fluxes[i].flux[im] = np.sum(s.flux * f.response) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2)) else: fluxes[i].flux[im, :] = np.sum(s.flux * f.response, axis=1) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2, axis=1)) for i, f in enumerate(binned_filters): fluxes[i].sort_to_match(par_table['MODEL_NAME']) fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits', overwrite=overwrite)
def build(self, age, sfh, dust, metal, fesc=1., sfh_law=exponential, dust_model=Calzetti, neb_dust_weight=1., neb_cont=True, neb_met=True, timesteps=500, verbose=False): """ Build composite stellar population SED(s) from input SSP models Parameters ---------- ssp : SSP class age : '~astropy.units.Quantity' array (with units of time) Desired stellar population age(s) since the onset of star-formation sfh : array of float or '~astropy.units.Quantity', Star-formation history parameter/exponent dust : array of floats, Strength of dust extinction in Av metal_ind : float or float array, Metallicity or metallicity range of stellar population relative to solar metallicity (Z/Z_sol), min and max allowed values set by range of input metallicities in 'ssp' class f_esc : float or array of float, Escape fraction(s) of nebular emission component sfh_law : '~smpy.sfh' or user defined function, Star-formation history parametrisation for composite stellar population. dust_model : '~smpy.dust' or user defined function, Dust extinction or attenuation law parametrisation neb_cont : boolean, default = True Include continuum emission component in nebular emission model neb_met : boolean, default = True Include metal line component in nebular emission model Returns ------- """ try: self.tau = u.Quantity(sfh, ndmin=1) except: self.tau = sfh self.tg = u.Quantity(age, ndmin=1).to(u.yr) self.tauv = np.array(dust, ndmin=1) self.mi = np.array(metal, ndmin=1) self.fesc = np.array(fesc, ndmin=1) self.sfh_law = sfh_law self.inc_cont = neb_cont self.inc_met = neb_met self.getAttenuation = dust_model self.sfr_func = sfh_law mu = 0.3 self.ta = self.ages outshape = [ len(self.mi), len(self.tg), len(self.tau), len(self.tauv), len(self.fesc), len(self.wave) ] self.SED = np.zeros(outshape) * self.sed_arr.unit self.STR = np.zeros(self.SED.shape[:-1]) self.SFR = np.zeros(self.SED.shape[:-1]) * u.solMass / u.yr # Set up nebular emission arrays -- WILL CHANGE if len(self.neb_wave) != len(self.wave): self.neb_cont = griddata(self.neb_wave, self.neb_cont, self.wave) self.neb_hlines = griddata(self.neb_wave, self.neb_hlines, self.wave) neb_metaln = np.zeros((len(self.wave), 3)) for i in range(3): neb_metaln[:, i] = griddata(self.neb_wave, self.neb_metal[:, i], self.wave) self.neb_metal = neb_metaln self.neb_wave = self.wave self.neb_cont[self.wave <= 912. * u.AA] = 0. self.neb_hlines[self.wave <= 912. * u.AA] = 0. self.neb_metal[self.wave <= 912. * u.AA, :] = 0. self.Nly_arr = self.calc_lyman(self.wave, self.sed_arr) self.neb_sed_arr = np.ones_like(self.sed_arr.value) nebrange1 = (self.metallicities <= 0.02) nebrange2 = (self.metallicities > 0.02) * (self.metallicities <= 0.2) nebrange3 = (self.metallicities > 0.2) self.neb_sed_arr[nebrange1, :, :] *= ( (self.neb_cont * self.inc_cont) + self.neb_hlines + (self.neb_metal[:, 0] * self.inc_met)) self.neb_sed_arr[nebrange2, :, :] *= ( (self.neb_cont * self.inc_cont) + self.neb_hlines + (self.neb_metal[:, 1] * self.inc_met)) self.neb_sed_arr[nebrange3, :, :] *= ( (self.neb_cont * self.inc_cont) + self.neb_hlines + (self.neb_metal[:, 2] * self.inc_met)) self.neb_sed_arr *= (c.c.to(u.AA / u.s) / (self.wave**2)).value # Convert to Flambda self.neb_sed_arr = (self.neb_sed_arr * self.sed_arr.unit / self.Nly_arr.unit) # Set up grid for ND-interpolation ti, mi = np.meshgrid( np.log10(self.ages / u.yr).value, np.log10(self.metallicities)) self.grid = zip(mi.flatten(), ti.flatten()) tri_grid = spatial.Delaunay(self.grid) # Make cube of interpolated age and SFHs. # Uses array slicing and vectorisation to minimise # loops where possible. sfh_grid_shape = (len(self.mi), timesteps) self.me_sfh = np.ones(sfh_grid_shape) for met_idx, metal in enumerate(self.mi): self.me_sfh[met_idx] *= metal if verbose: bar = ProgressBar(np.product(self.SED.shape[1:-1])) for idG, age in enumerate(self.tg): ta_range = np.logspace( np.log10(self.ages / u.yr).min(), np.log10(age / u.yr), timesteps) self.ta_sfh = np.ones(sfh_grid_shape) * u.yr self.ta_sfh *= ta_range[None, :] # Calculate Barycentric coordinates for all ages/metallicities in SFH. points = np.array( zip(np.log10(self.me_sfh.flatten()), np.log10(self.ta_sfh.flatten() / u.yr))) #if verbose: # print 'Interpolating SEDs at SFH timesteps' ss = tri_grid.find_simplex(points) X = tri_grid.transform[ss, :2] Y = points - tri_grid.transform[ss, 2] b = np.einsum('ijk,ik->ij', X, Y) self.bc = np.c_[b, 1 - b.sum(axis=1)] self.simplices = tri_grid.simplices[ss] #if verbose: # print 'Reshaping grids' # Interpolate SED, stellar mass fraction and remnant fractions # for SFH age grid using calculated Barycentric coordinates (bc). #print self.sed_arr.shape #print (len(self.metallicities) * len(self.ages), self.iw) self.temp = self.sed_arr.reshape( len(self.metallicities) * len(self.ages), self.iw)[self.simplices] self.sed_sfh = (self.temp * self.bc[:, :, None]).sum(1) self.sed_sfh = self.sed_sfh.reshape( np.append(sfh_grid_shape, self.iw)) self.temp = self.neb_sed_arr.reshape( len(self.metallicities) * len(self.ages), self.iw)[self.simplices] self.neb_sed_sfh = (self.temp * self.bc[:, :, None]).sum(1) self.neb_sed_sfh = self.neb_sed_sfh.reshape( np.append(sfh_grid_shape, self.iw)) self.strm_sfh = np.array( self.strm_arr.reshape( len(self.metallicities) * len(self.ages))[self.simplices] * self.bc).sum(1) self.strm_sfh = self.strm_sfh.reshape(sfh_grid_shape) self.rmtm_sfh = np.array( self.rmtm_arr.reshape( len(self.metallicities) * len(self.ages))[self.simplices] * self.bc).sum(1) self.rmtm_sfh = self.rmtm_sfh.reshape(sfh_grid_shape) self.Nly_sfh = (self.Nly_arr.reshape( len(self.metallicities) * len(self.ages))[self.simplices] * self.bc).sum(1) self.Nly_sfh = self.Nly_sfh.reshape(sfh_grid_shape) #* #(1 - self.fesc[None, None, :, None])) # Star-formation history for idT, t in enumerate(self.tau): if type(t) == tuple: tau = t else: tau = tuple([t]) self.sfr_hist = self.sfr_func(self.ta_sfh, *tau) # Enforce integrated SFR = 1 Msol. self.norm = np.trapz(self.sfr_hist, self.ta_sfh, axis=-1)[:, None] self.sfr_hist /= self.norm self.weights = np.abs(self.sfr_func(self.tg[None, idG, None] \ - self.ta_sfh, *tau)) / self.norm self.sfh_weights = np.ones(sfh_grid_shape) * self.weights for idA, Av in enumerate(self.tauv): for idf, fesc in enumerate(self.fesc): self.Att = self.getAttenuation(self.ta_sfh, \ self.wave, Av) neb_att = self.getAttenuation(self.ta_sfh, \ self.wave, Av * neb_dust_weight) combined_sed = np.copy( self.sed_sfh) * self.sed_sfh.unit # Absorbed LyC photons combined_sed[:, :, self.wave <= 912 * u.AA] *= fesc # Resulting nebular emission combined_sed *= self.Att # Dust attenuated combined SED combined_sed += ( neb_att * ((1 - fesc) * self.Nly_sfh[:, :, None] * self.neb_sed_sfh)) # Integrate over star-formation history self.SED[:, idG, idT, idA, idf, :] = \ np.trapz(self.sfh_weights[:, :, None] * \ combined_sed, self.ta_sfh[:, :, None], axis=-2) self.STR[:, idG, idT, idA, idf] = \ np.trapz(self.sfh_weights * self.strm_sfh, self.ta_sfh) self.SFR[:, idG, idT, idA, idf] = \ self.sfr_hist[:, -1] * u.solMass if verbose: bar.update() # Normalise by stellar mass fraction (stars + remnants in case of BC03) self.SFR = self.SFR / self.STR self.SED = self.SED / self.STR[:, :, :, :, :, None] self.Ms = np.ones_like(self.SFR.value) * u.Msun self.Nly = self.calc_lyman(self.wave, self.SED).cgs
def render_chem(yth2co321=yth2co321, yth2co303=yth2co303, ytsio=ytsio, outdir='yt_renders_chem3', size=512, scale=1100., nframes=60, north_vector=[1, 0, 0], rot_vector=[1, 0, 0], movie=True, camera_angle=[-0.6, 0.4, 0.6]): if not os.path.exists(paths.mpath(outdir)): os.makedirs(paths.mpath(outdir)) tf1 = yt.ColorTransferFunction([0.1, 2], grey_opacity=True) #tf1.add_gaussian(0.1,0.05, [1,0,0,1]) #tf1.map_to_colormap(0, 0.5, scale=1, colormap='Reds') tf1.add_gaussian(0.25, 0.01, [1, 0, 0, 1]) tf1.add_step(0.25, 0.5, [1, 0, 0, 1]) #tf1.add_step(1,2,[1,1,1,1]) tf1.map_to_colormap(0.5, 2, scale=1, colormap=red) tf2 = yt.ColorTransferFunction([0.1, 2], grey_opacity=True) #tf2.add_gaussian(0.1,0.05, [0,1,0,1]) #tf2.map_to_colormap(0, 0.5, scale=1, colormap='Greens') tf2.add_gaussian(0.25, 0.01, [0, 1, 0, 1]) tf2.add_step(0.25, 0.5, [0, 1, 0, 1]) tf2.map_to_colormap(0.5, 2, scale=1, colormap=green) tf3 = yt.ColorTransferFunction([0.1, 2], grey_opacity=True) #tf3.add_gaussian(0.1,0.05, [0,0,1,1]) #tf3.map_to_colormap(0, 0.5, scale=1, colormap='Blues') tf3.add_gaussian(0.25, 0.01, [0, 0, 1, 1]) tf3.add_step(0.25, 0.5, [0, 0, 1, 1]) tf3.map_to_colormap(0.5, 2, scale=1, colormap=blue) center = yth2co303.dataset.domain_dimensions / 2. camh2co303 = yth2co303.dataset.h.camera(center, camera_angle, scale, size, tf3, north_vector=north_vector, fields='flux') camh2co321 = yth2co321.dataset.h.camera(center, camera_angle, scale, size, tf2, north_vector=north_vector, fields='flux') camsio = ytsio.dataset.h.camera(center, camera_angle, scale, size, tf1, north_vector=north_vector, fields='flux') imh2co303 = camh2co303.snapshot() imh2co321 = camh2co321.snapshot() imsio = camsio.snapshot() pl.figure(1) pl.clf() pl.imshow(imh2co303 + imh2co321 + imsio) pl.figure(2) pl.clf() pl.imshow(imh2co303[:, :, :3] + imh2co321[:, :, :3] + imsio[:, :, :3]) if movie: images_h2co303 = [imh2co303] images_h2co321 = [imh2co321] images_sio = [imsio] r1 = camh2co303.rotation(2 * np.pi, nframes, rot_vector=rot_vector) r2 = camh2co321.rotation(2 * np.pi, nframes, rot_vector=rot_vector) r3 = camsio.rotation(2 * np.pi, nframes, rot_vector=rot_vector) pb = ProgressBar(nframes * 3) for (ii, (imh2co303, imh2co321, imsio)) in enumerate(izip(r1, r2, r3)): images_h2co303.append(imh2co303) images_h2co321.append(imh2co321) images_sio.append(imsio) imh2co303 = imh2co303.swapaxes(0, 1) imh2co303.write_png(paths.mpath( os.path.join(outdir, "h2co303_%04i.png" % (ii))), rescale=False) pb.update(ii * 3) imsio = imsio.swapaxes(0, 1) imsio.write_png(paths.mpath( os.path.join(outdir, "sio_%04i.png" % (ii))), rescale=False) pb.update(ii * 3 + 1) imh2co321 = imh2co321.swapaxes(0, 1) imh2co321.write_png(paths.mpath( os.path.join(outdir, "h2co321_%04i.png" % (ii))), rescale=False) pb.update(ii * 3 + 2) pb.next() save_images([ i1 + i2 + i3 for i1, i2, i3 in izip(images_h2co303, images_h2co321, images_sio) ], paths.mpath(outdir)) make_movie(paths.mpath(outdir)) return images_h2co303, images_h2co321, images_sio else: return imh2co303, imh2co321, imsio
def build(self, SED, Filters, redshift, savepath, force_age=True, madau=True, units=u.uJy, verbose=False, clobber=True): """ Parameters ---------- SED : '~smpy.CSP' object Built CSP model set Filters : '~smpy.FilterSet' object Filter set through which to observe the set of models included in SED object redshift : array Redshift(s) at which to observe SED set savepath : str Filename for hdf5 save file force_age : boolean Require age of the stellar population to be younger than the age of the Universe at the desired redshift. madau : boolean Apply IGM absorption following Madau 1999 prescription units : '~astropy.units.Quantity' Desired output units, must be in spectral flux density equivalent verbose : boolean Add additional terminal outputs if true """ self.F = Filters self.redshifts = np.array(redshift, ndmin=1) self.wave = SED.wave #self.fluxes = np.zeros(gridshape) * units #self.AB = np.zeros(gridshape) * u.mag self.wl = np.zeros(len(self.F.filters)) * u.AA self.fwhm = np.zeros(len(self.F.filters)) * u.AA self.dl = cosmo.luminosity_distance(self.redshifts).cgs self.dl[self.redshifts == 0] = 10 * c.pc assert type( savepath ) is types.StringType, "File save path is not a string: %r" % savepath self.savepath = savepath if clobber: if os.path.isfile(self.savepath): os.remove(self.savepath) with h5py.File(savepath, 'w') as f: gridshape = np.append( [len(self.redshifts), len(self.F.filters)], SED.SED.shape[:-1]) self.fluxes = f.create_dataset("fluxes", gridshape, dtype='f') self.fluxes.attrs['unit'] = units.to_string() self.AB = f.create_dataset("mags", gridshape, dtype='f') for j, filt in enumerate(self.F.filters): self.wl[j] = filt.lambda_c self.fwhm[j] = filt.fwhm print('Filter {0}' \ '(Central WL = {1:.1f}):'.format(j+1, filt.lambda_c)) # Find SED wavelength entries within filter range wff = np.logical_and(filt.wave[0] < self.wave, self.wave < filt.wave[-1]) wft = self.wave[wff] # Interpolate to find throughput values at new wavelength points tpt = griddata(filt.wave, filt.response, wft) # Join arrays and sort w.r.t to wf # Also replace units stripped by concatenate wf = np.array(np.concatenate((filt.wave, wft))) * u.AA tp = np.concatenate((filt.response, tpt)) order = np.argsort(wf) wf = wf[order] tp = tp[order] if verbose: niters = len(self.redshifts) * len(SED.tg) bar = ProgressBar(niters) for i, z in enumerate(self.redshifts): self.lyman_abs = np.ones(len(self.wave)) if madau: self.lyman_abs = np.clip(tau_madau(self.wave, z), 0., 1.) for a, age in enumerate(SED.tg): if np.logical_or(age < cosmo.age(z), force_age): fluxes = self.calcflux(SED.SED[:, a], wf, tp, z, self.dl[i], units) self.fluxes[i, j, :, a] = fluxes.to(units) else: # Set fluxes for ages older # than universe to zero self.fluxes[i, j, :, a] = 0. if verbose: assert isinstance(bar, object) bar.update() print('\n') for i, z in enumerate(self.redshifts): for j, filt in enumerate(self.F.filters): self.AB[i, j] = (-2.5 * np.log10( (self.fluxes[i, j] * units).to(u.Jy) / (3631 * u.Jy))).value zeros = (self.fluxes[i, j] == 0.) self.AB[i, j][zeros] = np.inf # Set up hdf5 dimensions based on SED ranges for convenient # slicing if not used for fitting. f.create_dataset('z', data=self.redshifts) f.create_dataset('ages', data=SED.tg.value) f['ages'].attrs['unit'] = SED.tg.unit.to_string() f.create_dataset('dust', data=SED.tauv) f.create_dataset('metallicities', data=SED.mi) f.create_dataset('fesc', data=SED.fesc) try: s = SED.taus.shape taus = f.create_dataset('sfh', data=SED.tau) taus.attrs['unit'] = SED.tau.unit.to_string() except: # For multi-parameter SFHs, use indices for scale taus = f.create_dataset('sfh', data=np.arange(len(SED.tau))) for dataset in ['fluxes', 'mags']: f[dataset].dims.create_scale(f['z'], 'z') f[dataset].dims.create_scale(f['sfh'], 'sfh') f[dataset].dims.create_scale(f['ages'], 'ages') f[dataset].dims.create_scale(f['dust'], 'dust') f[dataset].dims.create_scale(f['metallicities'], 'met') f[dataset].dims.create_scale(f['fesc'], 'fesc') f[dataset].dims[0].attach_scale(f['z']) f[dataset].dims[2].attach_scale(f['metallicities']) f[dataset].dims[3].attach_scale(f['ages']) f[dataset].dims[4].attach_scale(f['sfh']) f[dataset].dims[5].attach_scale(f['dust']) f[dataset].dims[6].attach_scale(f['fesc']) f.create_dataset('wl', data=self.wl) f.create_dataset('fwhm', data=self.fwhm) # Store all CSP attributes in case they are needed later for attribute in SED.__dict__.keys(): if attribute == 'SED': # SED excluded due to large size continue try: f.create_dataset(attribute, data=SED.__dict__[attribute]) except: # in cases of functions etc. continue self.load(self.savepath)
def compute_bispectrum(self, show_progress=True, use_pyfftw=False, threads=1, nsamples=100, seed=1000, mean_subtract=False, **pyfftw_kwargs): ''' Do the computation. Parameters ---------- show_progress : optional, bool Show progress bar while sampling the bispectrum. use_pyfftw : bool, optional Enable to use pyfftw, if it is installed. threads : int, optional Number of threads to use in FFT when using pyfftw. nsamples : int, optional Sets the number of samples to take at each vector magnitude. seed : int, optional Sets the seed for the distribution draws. mean_subtract : bool, optional Subtract the mean from the data before computing. This removes the "zero frequency" (i.e., constant) portion of the power, resulting in a loss of phase coherence along the k_1=k_2 line. pyfft_kwargs : Passed to `~turbustat.statistics.rfft_to_fft.rfft_to_fft`. See `here <https://hgomersall.github.io/pyFFTW/pyfftw/interfaces/interfaces.html#interfaces-additional-args>`_ for a list of accepted kwargs. ''' if mean_subtract: norm_data = self.data - self.data.mean() else: norm_data = self.data if use_pyfftw: if PYFFTW_FLAG: if pyfftw_kwargs.get('threads') is not None: pyfftw_kwargs.pop('threads') fftarr = fft2(norm_data, threads=threads, **pyfftw_kwargs) else: warn("pyfftw not installed. Reverting to using numpy.") use_pyfftw = False if not use_pyfftw: fftarr = np.fft.fft2(norm_data) conjfft = np.conj(fftarr) bispec_shape = (int(self.shape[0] / 2.), int(self.shape[1] / 2.)) self._bispectrum = np.zeros(bispec_shape, dtype=np.complex) self._bicoherence = np.zeros(bispec_shape, dtype=np.float) self._tracker = np.zeros(self.shape, dtype=np.int16) biconorm = np.ones_like(self.bispectrum, dtype=float) if show_progress: bar = ProgressBar(np.prod(fftarr.shape) / 4.) prod = product(range(int(fftarr.shape[0] / 2.)), range(int(fftarr.shape[1] / 2.))) with NumpyRNGContext(seed): for n, (k1mag, k2mag) in enumerate(prod): phi1 = ra.uniform(0, 2 * np.pi, nsamples) phi2 = ra.uniform(0, 2 * np.pi, nsamples) k1x = np.asarray( [int(k1mag * np.cos(angle)) for angle in phi1]) k2x = np.asarray( [int(k2mag * np.cos(angle)) for angle in phi2]) k1y = np.asarray( [int(k1mag * np.sin(angle)) for angle in phi1]) k2y = np.asarray( [int(k2mag * np.sin(angle)) for angle in phi2]) k3x = np.asarray([ int(k1mag * np.cos(ang1) + k2mag * np.cos(ang2)) for ang1, ang2 in zip(phi1, phi2) ]) k3y = np.asarray([ int(k1mag * np.sin(ang1) + k2mag * np.sin(ang2)) for ang1, ang2 in zip(phi1, phi2) ]) samps = fftarr[k1x, k1y] * fftarr[k2x, k2y] * conjfft[k3x, k3y] self._bispectrum[k1mag, k2mag] = np.sum(samps) biconorm[k1mag, k2mag] = np.sum(np.abs(samps)) # Track where we're sampling from in fourier space self._tracker[k1x, k1y] += 1 self._tracker[k2x, k2y] += 1 self._tracker[k3x, k3y] += 1 if show_progress: bar.update(n + 1) self._bicoherence = (np.abs(self.bispectrum) / biconorm) self._bispectrum_amp = np.log10(np.abs(self.bispectrum))
def convolve_model_dir_monochromatic(model_dir, overwrite=False, max_ram=8, wav_min=-np.inf * u.micron, wav_max=np.inf * u.micron): """ Convolve all the model SEDs in a model directory Parameters ---------- model_dir : str The path to the model directory overwrite : bool, optional Whether to overwrite the output files max_ram : float, optional The maximum amount of RAM that can be used (in Gb) wav_min : float, optional The minimum wavelength to consider. Only wavelengths above this value will be output. wav_max : float, optional The maximum wavelength to consider. Only wavelengths below this value will be output. """ modpar = parfile.read(os.path.join(model_dir, 'models.conf'), 'conf') if modpar.get('version', 1) > 1: raise ValueError("monochromatic filters are no longer used for new-style model directories") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') + glob.glob(model_dir + '/seds/*/*.fits.gz') + glob.glob(model_dir + '/seds/*.fits') + glob.glob(model_dir + '/seds/*/*.fits')) par_table = load_parameter_table(model_dir) # Find number of models n_models = len(sed_files) if n_models == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(n_models, model_dir)) # Find out apertures and wavelengths first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures n_wav = first_sed.n_wav wavelengths = first_sed.wav # For model grids that are very large, it is not possible to compute all # fluxes in one go, so we need to process in chunks in wavelength space. chunk_size = min(n_wav, int(np.floor(max_ram * 1024. ** 3 / (4. * 2. * n_models * n_ap)))) if chunk_size == n_wav: log.info("Producing all monochromatic files in one go") else: log.info("Producing monochromatic files in chunks of {0}".format(chunk_size)) filters = Table() filters['wav'] = wavelengths filters['filter'] = np.zeros(wavelengths.shape, dtype='S10') # Figure out range of wavelength indices to use # (wavelengths array is sorted in reverse order) jlo = n_wav - 1 - (wavelengths[::-1].searchsorted(wav_max) - 1) jhi = n_wav - 1 - wavelengths[::-1].searchsorted(wav_min) chunk_size = min(chunk_size, jhi - jlo + 1) # Loop over wavelength chunks for jmin in range(jlo, jhi, chunk_size): # Find upper wavelength to compute jmax = min(jmin + chunk_size - 1, jhi) log.info('Processing wavelengths {0} to {1}'.format(jmin, jmax)) # Set up convolved fluxes fluxes = [ConvolvedFluxes(model_names=np.zeros(n_models, dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(chunk_size)] b = ProgressBar(len(sed_files)) # Loop over SEDs for im, sed_file in enumerate(sed_files): b.update() log.debug('Processing {0}'.format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu') # Convolve for j in range(chunk_size): fluxes[j].central_wavelength = wavelengths[j + jmin] fluxes[j].apertures = apertures fluxes[j].model_names[im] = s.name if n_ap == 1: fluxes[j].flux[im] = s.flux[0, j + jmin] fluxes[j].error[im] = s.error[0, j + jmin] else: fluxes[j].flux[im, :] = s.flux[:, j + jmin] fluxes[j].error[im, :] = s.error[:, j + jmin] for j in range(chunk_size): fluxes[j].sort_to_match(par_table['MODEL_NAME']) fluxes[j].write('{0:s}/convolved/MO{1:03d}.fits'.format(model_dir, j + jmin + 1), overwrite=overwrite) filters['filter'][j + jmin] = "MO{0:03d}".format(j + jmin + 1) return filters
def compute_deltavar(self, allow_huge=False, boundary='wrap', min_weight_frac=0.01, nan_treatment='fill', preserve_nan=False, use_pyfftw=False, threads=1, pyfftw_kwargs={}, show_progress=True, keep_convolve_arrays=False): ''' Perform the convolution and calculate the delta variance at all lags. Parameters ---------- allow_huge : bool, optional Passed to `~astropy.convolve.convolve_fft`. Allows operations on images larger than 1 Gb. boundary : {"wrap", "fill"}, optional Use "wrap" for periodic boundaries, and "fill" for non-periodic. min_weight_frac : float, optional Set the fraction of the peak of the weight array to mask below. Default is 0.01. This will remove most edge artifacts, but is not guaranteed to! Increase this value if artifacts are encountered (this typically results in large spikes in the delta-variance curve). nan_treatment : bool, optional Enable to interpolate over NaNs in the convolution. Default is True. use_pyfftw : bool, optional Enable to use pyfftw, if it is installed. threads : int, optional Number of threads to use in FFT when using pyfftw. pyfftw_kwargs : Passed to See `here <http://hgomersall.github.io/pyFFTW/pyfftw/builders/builders.html>`_ for a list of accepted kwargs. show_progress : bool, optional Show a progress bar while convolving the image at each lag. keep_convolve_arrays : bool, optional Keeps the convolved arrays at each lag. Disabled by default to minimize memory usage. ''' self._delta_var = np.empty((len(self.lags))) self._delta_var_error = np.empty((len(self.lags))) if show_progress: bar = ProgressBar(len(self.lags)) for i, lag in enumerate(self.lags.value): core = core_kernel(lag, self.data.shape[0], self.data.shape[1]) annulus = annulus_kernel(lag, self.diam_ratio, self.data.shape[0], self.data.shape[1]) if boundary == "wrap": # Don't pad for periodic boundaries pad_weights = self.weights pad_img = self.data * self.weights elif boundary == "fill": # Extend to avoid boundary effects from non-periodicity pad_weights = np.pad(self.weights, int(lag), padwithzeros) pad_img = np.pad(self.data, int(lag), padwithzeros) * \ pad_weights else: raise ValueError("boundary must be 'wrap' or 'fill'. " "Given {}".format(boundary)) img_core = \ convolution_wrapper(pad_img, core, boundary=boundary, fill_value=np.NaN, allow_huge=allow_huge, nan_treatment=nan_treatment, use_pyfftw=use_pyfftw, threads=threads, pyfftw_kwargs=pyfftw_kwargs) img_annulus = \ convolution_wrapper(pad_img, annulus, boundary=boundary, fill_value=np.NaN, allow_huge=allow_huge, nan_treatment=nan_treatment, use_pyfftw=use_pyfftw, threads=threads, pyfftw_kwargs=pyfftw_kwargs) weights_core = \ convolution_wrapper(pad_weights, core, boundary=boundary, fill_value=np.NaN, allow_huge=allow_huge, nan_treatment=nan_treatment, use_pyfftw=use_pyfftw, threads=threads, pyfftw_kwargs=pyfftw_kwargs) weights_annulus = \ convolution_wrapper(pad_weights, annulus, boundary=boundary, fill_value=np.NaN, allow_huge=allow_huge, nan_treatment=nan_treatment, use_pyfftw=use_pyfftw, threads=threads, pyfftw_kwargs=pyfftw_kwargs) cutoff_val = min_weight_frac * self.weights.max() weights_core[np.where(weights_core <= cutoff_val)] = np.NaN weights_annulus[np.where(weights_annulus <= cutoff_val)] = np.NaN conv_arr = (img_core / weights_core) - \ (img_annulus / weights_annulus) conv_weight = weights_core * weights_annulus if keep_convolve_arrays: self._convolved_arrays.append(conv_arr) self._convolved_weights.append(weights_core * weights_annulus) val, err = _delvar(conv_arr, conv_weight, lag) if (val <= 0) or (err <= 0) or np.isnan(val) or np.isnan(err): self._delta_var[i] = np.NaN self._delta_var_error[i] = np.NaN else: self._delta_var[i] = val self._delta_var_error[i] = err if show_progress: bar.update(i + 1)
def measure_dendrogram_properties(dend=None, cube303=cube303, cube321=cube321, cube13co=cube13co, cube18co=cube18co, noise_cube=noise_cube, sncube=sncube, suffix="", last_index=None, plot_some=True, line='303', write=True): assert (cube321.shape == cube303.shape == noise_cube.shape == cube13co.shape == cube18co.shape == sncube.shape) assert sncube.wcs is cube303.wcs is sncube.mask._wcs metadata = {} metadata['data_unit'] = u.K metadata['spatial_scale'] = 7.2 * u.arcsec metadata['beam_major'] = 30 * u.arcsec metadata['beam_minor'] = 30 * u.arcsec metadata['wavelength'] = 218.22219*u.GHz metadata['velocity_scale'] = u.km/u.s metadata['wcs'] = cube303.wcs keys = [ 'density_chi2', 'expected_density', 'dmin1sig_chi2', 'dmax1sig_chi2', 'column_chi2', 'expected_column', 'cmin1sig_chi2', 'cmax1sig_chi2', 'temperature_chi2', 'expected_temperature', 'tmin1sig_chi2', 'tmax1sig_chi2', 'eratio321303', 'ratio321303', 'logh2column', 'elogh2column', 'logabundance', 'elogabundance', ] obs_keys = [ 'Stot303', 'Smin303', 'Smax303', 'Stot321', 'Smean303', 'Smean321', 'npix', 'e303', 'e321', 'r321303', 'er321303', '13cosum', 'c18osum', '13comean', 'c18omean', 's_ntotal', 'index', 'is_leaf', 'parent', 'root', 'lon', 'lat', 'vcen', 'higaldusttem', 'reff', 'dustmass', 'dustmindens', 'bad', #'tkin_turb', ] columns = {k:[] for k in (keys+obs_keys)} log.debug("Initializing dendrogram temperature fitting loop") # FORCE wcs to match # (technically should reproject here) cube13co._wcs = cube18co._wcs = cube303.wcs cube13co.mask._wcs = cube18co.mask._wcs = cube303.wcs if line == '303': maincube = cube303 elif line == '321': maincube = cube321 else: raise ValueError("Unrecognized line: {0}".format(line)) # Prepare an array to hold the fitted temperatures tcubedata = np.empty(maincube.shape, dtype='float32') tcubedata[:] = np.nan tcubeleafdata = np.empty(maincube.shape, dtype='float32') tcubeleafdata[:] = np.nan nbad = 0 catalog = ppv_catalog(dend, metadata) pb = ProgressBar(len(catalog)) for ii,row in enumerate(catalog): structure = dend[row['_idx']] assert structure.idx == row['_idx'] == ii dend_obj_mask = BooleanArrayMask(structure.get_mask(), wcs=cube303.wcs) dend_inds = structure.indices() view = (slice(dend_inds[0].min(), dend_inds[0].max()+1), slice(dend_inds[1].min(), dend_inds[1].max()+1), slice(dend_inds[2].min(), dend_inds[2].max()+1),) #view2 = cube303.subcube_slices_from_mask(dend_obj_mask) submask = dend_obj_mask[view] #assert np.count_nonzero(submask.include()) == np.count_nonzero(dend_obj_mask.include()) sn = sncube[view].with_mask(submask) sntot = sn.sum().value #np.testing.assert_almost_equal(sntot, structure.values().sum(), decimal=0) c303 = cube303[view].with_mask(submask) c321 = cube321[view].with_mask(submask) co13sum = cube13co[view].with_mask(submask).sum().value co18sum = cube18co[view].with_mask(submask).sum().value if hasattr(co13sum,'__len__'): raise TypeError(".sum() applied to an array has yielded a non scalar.") npix = submask.include().sum() assert npix == structure.get_npix() Stot303 = c303.sum().value if np.isnan(Stot303): raise ValueError("NaN in cube. This can't happen: the data from " "which the dendrogram was derived can't have " "NaN pixels.") Smax303 = c303.max().value Smin303 = c303.min().value Stot321 = c321.sum().value if npix == 0: raise ValueError("npix=0. This is impossible.") Smean303 = Stot303/npix if Stot303 <= 0 and line=='303': raise ValueError("The 303 flux is <=0. This isn't possible because " "the dendrogram was derived from the 303 data with a " "non-zero threshold.") elif Stot303 <= 0 and line=='321': Stot303 = 0 Smean303 = 0 elif Stot321 <= 0 and line=='321': raise ValueError("The 321 flux is <=0. This isn't possible because " "the dendrogram was derived from the 321 data with a " "non-zero threshold.") if np.isnan(Stot321): raise ValueError("NaN in 321 line") Smean321 = Stot321/npix #error = (noise_cube[view][submask.include()]).sum() / submask.include().sum()**0.5 var = ((noise_cube[dend_obj_mask.include()]**2).sum() / npix**2) error = var**0.5 if np.isnan(error): raise ValueError("error is nan: this is impossible by definition.") if line == '321' and Stot303 == 0: r321303 = np.nan er321303 = np.nan elif Stot321 < 0: r321303 = error / Smean303 er321303 = (r321303**2 * (var/Smean303**2 + 1))**0.5 else: r321303 = Stot321 / Stot303 er321303 = (r321303**2 * (var/Smean303**2 + var/Smean321**2))**0.5 for c in columns: assert len(columns[c]) == ii columns['index'].append(row['_idx']) columns['s_ntotal'].append(sntot) columns['Stot303'].append(Stot303) columns['Smax303'].append(Smax303) columns['Smin303'].append(Smin303) columns['Stot321'].append(Stot321) columns['Smean303'].append(Smean303) columns['Smean321'].append(Smean321) columns['npix'].append(npix) columns['e303'].append(error) columns['e321'].append(error) columns['r321303'].append(r321303) columns['er321303'].append(er321303) columns['13cosum'].append(co13sum) columns['c18osum'].append(co18sum) columns['13comean'].append(co13sum/npix) columns['c18omean'].append(co18sum/npix) columns['is_leaf'].append(structure.is_leaf) columns['parent'].append(structure.parent.idx if structure.parent else -1) columns['root'].append(get_root(structure).idx) s_main = maincube._data[dend_inds] x,y,z = maincube.world[dend_inds] lon = ((z.value-(360*(z.value>180)))*s_main).sum()/s_main.sum() lat = (y*s_main).sum()/s_main.sum() vel = (x*s_main).sum()/s_main.sum() columns['lon'].append(lon) columns['lat'].append(lat.value) columns['vcen'].append(vel.value) mask2d = dend_obj_mask.include().max(axis=0)[view[1:]] logh2column = np.log10(np.nanmean(column_regridded.data[view[1:]][mask2d]) * 1e22) if np.isnan(logh2column): log.info("Source #{0} has NaNs".format(ii)) logh2column = 24 elogh2column = elogabundance columns['higaldusttem'].append(np.nanmean(dusttem_regridded.data[view[1:]][mask2d])) r_arcsec = row['radius']*u.arcsec reff = (r_arcsec*(8.5*u.kpc)).to(u.pc, u.dimensionless_angles()) mass = ((10**logh2column*u.cm**-2)*np.pi*reff**2*2.8*constants.m_p).to(u.M_sun) density = (mass/(4/3.*np.pi*reff**3)/constants.m_p/2.8).to(u.cm**-3) columns['reff'].append(reff.value) columns['dustmass'].append(mass.value) columns['dustmindens'].append(density.value) mindens = np.log10(density.value) if mindens < 3: mindens = 3 if (r321303 < 0 or np.isnan(r321303)) and line != '321': raise ValueError("Ratio <0: This can't happen any more because " "if either num/denom is <0, an exception is " "raised earlier") #for k in columns: # if k not in obs_keys: # columns[k].append(np.nan) elif (r321303 < 0 or np.isnan(r321303)) and line == '321': for k in keys: columns[k].append(np.nan) else: # Replace negatives for fitting if Smean321 <= 0: Smean321 = error mf.set_constraints(ratio321303=r321303, eratio321303=er321303, #ratio321322=ratio2, eratio321322=eratio2, logh2column=logh2column, elogh2column=elogh2column, logabundance=logabundance, elogabundance=elogabundance, taline303=Smean303, etaline303=error, taline321=Smean321, etaline321=error, mindens=mindens, linewidth=10) row_data = mf.get_parconstraints() row_data['ratio321303'] = r321303 row_data['eratio321303'] = er321303 for k in row_data: columns[k].append(row_data[k]) # Exclude bad velocities from cubes if row['v_cen'] < -80e3 or row['v_cen'] > 180e3: # Skip: there is no real structure down here nbad += 1 is_bad = True else: is_bad = False tcubedata[dend_obj_mask.include()] = row_data['expected_temperature'] if structure.is_leaf: tcubeleafdata[dend_obj_mask.include()] = row_data['expected_temperature'] columns['bad'].append(is_bad) width = row['v_rms']*u.km/u.s lengthscale = reff #REMOVED in favor of despotic version done in dendrograms.py # we use the analytic version here; the despotic version is # computed elsewhere (with appropriate gcor factors) #columns['tkin_turb'].append(heating.tkin_all(10**row_data['density_chi2']*u.cm**-3, # width, # lengthscale, # width/lengthscale, # columns['higaldusttem'][-1]*u.K, # crir=0./u.s)) if len(set(len(c) for k,c in columns.items())) != 1: print("Columns are different lengths. This is not allowed.") import ipdb; ipdb.set_trace() for c in columns: assert len(columns[c]) == ii+1 if plot_some and not is_bad and (ii-nbad % 100 == 0 or ii-nbad < 50): try: log.info("T: [{tmin1sig_chi2:7.2f},{expected_temperature:7.2f},{tmax1sig_chi2:7.2f}]" " R={ratio321303:8.4f}+/-{eratio321303:8.4f}" " Smean303={Smean303:8.4f} +/- {e303:8.4f}" " Stot303={Stot303:8.2e} npix={npix:6d}" .format(Smean303=Smean303, Stot303=Stot303, npix=npix, e303=error, **row_data)) pl.figure(1) pl.clf() mf.denstemplot() pl.savefig(fpath("dendrotem/diagnostics/{0}_{1}.png".format(suffix,ii))) pl.figure(2).clf() mf.parplot1d_all(levels=[0.68268949213708585]) pl.savefig(fpath("dendrotem/diagnostics/1dplot{0}_{1}.png".format(suffix,ii))) pl.draw() pl.show() except Exception as ex: print ex pass else: pb.update(ii+1) if last_index is not None and ii >= last_index: break if last_index is not None: catalog = catalog[:last_index+1] for k in columns: if k not in catalog.keys(): catalog.add_column(table.Column(name=k, data=columns[k])) for mid,lo,hi,letter in (('expected_temperature','tmin1sig_chi2','tmax1sig_chi2','t'), ('expected_density','dmin1sig_chi2','dmax1sig_chi2','d'), ('expected_column','cmin1sig_chi2','cmax1sig_chi2','c')): catalog.add_column(table.Column(name='elo_'+letter, data=catalog[mid]-catalog[lo])) catalog.add_column(table.Column(name='ehi_'+letter, data=catalog[hi]-catalog[mid])) if write: catalog.write(tpath('PPV_H2CO_Temperature{0}.ipac'.format(suffix)), format='ascii.ipac') # Note that there are overlaps in the catalog, which means that ORDER MATTERS # in the above loop. I haven't yet checked whether large scale overwrites # small or vice-versa; it may be that both views of the data are interesting. tcube = SpectralCube(data=tcubedata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit':'K'}, header=cube303.header, ) tcubeleaf = SpectralCube(data=tcubeleafdata, wcs=cube303.wcs, mask=cube303.mask, meta={'unit':'K'}, header=cube303.header, ) if write: log.info("Writing TemperatureCube") outpath = 'TemperatureCube_DendrogramObjects{0}.fits' tcube.write(hpath(outpath.format(suffix)), overwrite=True) outpath_leaf = 'TemperatureCube_DendrogramObjects{0}_leaves.fits' tcubeleaf.write(hpath(outpath_leaf.format(suffix)), overwrite=True) return catalog, tcube
def quick_render_movie(self, outdir, size=256, nframes=30, camera_angle=(0,0,1), north_vector=(0,0,1), rot_vector=(1,0,0), colormap='doom', cmap_range='auto', transfer_function='auto', start_index=0, image_prefix="", output_filename='out.mp4', log_scale=False, rescale=True): """ Create a movie rotating the cube 360 degrees from PP -> PV -> PP -> PV -> PP Parameters ---------- outdir: str The output directory in which the individual image frames and the resulting output mp4 file should be stored size: int The size of the individual output frame in pixels (i.e., size=256 will result in a 256x256 image) nframes: int The number of frames in the resulting movie camera_angle: 3-tuple The initial angle of the camera north_vector: 3-tuple The vector of 'north' in the data cube. Default is coincident with the spectral axis rot_vector: 3-tuple The vector around which the camera will be rotated colormap: str A valid colormap. See `yt.show_colormaps` transfer_function: 'auto' or `yt.visualization.volume_rendering.TransferFunction` Either 'auto' to use the colormap specified, or a valid TransferFunction instance log_scale: bool Should the colormap be log scaled? rescale: bool If True, the images will be rescaled to have a common 95th percentile brightness, which can help reduce flickering from having a single bright pixel in some projections start_index : int The number of the first image to save image_prefix : str A string to prepend to the image name for each image that is output output_filename : str The movie file name to output. The suffix may affect the file type created. Defaults to 'out.mp4'. Will be placed in ``outdir`` Returns ------- """ if not ytOK: raise IOError("yt could not be imported. Cube renderings are not possible.") scale = np.max(self.cube.shape) if not os.path.exists(outdir): os.makedirs(outdir) elif not os.path.isdir(outdir): raise OSError("Output directory {0} exists and is not a directory.".format(outdir)) if cmap_range == 'auto': upper = self.cube.max().value lower = self.cube.std().value * 3 cmap_range = [lower,upper] if transfer_function == 'auto': tfh = self.auto_transfer_function(cmap_range, log=log_scale) tfh.tf.map_to_colormap(cmap_range[0], cmap_range[1], colormap=colormap) tf = tfh.tf else: tf = transfer_function center = self.dataset.domain_center cam = self.dataset.h.camera(center, camera_angle, scale, size, tf, north_vector=north_vector, fields='flux') im = cam.snapshot() images = [im] pb = ProgressBar(nframes) for ii,im in enumerate(cam.rotation(2 * np.pi, nframes, rot_vector=rot_vector)): images.append(im) im.write_png(os.path.join(outdir,"%s%04i.png" % (image_prefix, ii+start_index)), rescale=False) pb.update(ii+1) log.info("Rendering complete in {0}s".format(time.time() - pb._start_time)) if rescale: _rescale_images(images, os.path.join(outdir, image_prefix)) pipe = _make_movie(outdir, prefix=image_prefix, filename=output_filename) return images
def render_13co( ytcube=yt13co, outdir='yt_renders_13CO', size=512, scale=1200., nframes=180, movie=True, camera_angle=[0, 0, 1], north_vector=[1, 0, 0], rot_vector1=[1, 0, 0], rot_vector2=[0.5, 0.5, 0.0], rot_vector3=[0.0, 0.5, 0.5], ): if not os.path.exists(paths.mpath(outdir)): os.makedirs(paths.mpath(outdir)) tf = yt.ColorTransferFunction([0, 30], grey_opacity=True) #tf.map_to_colormap(0.1,5,colormap='Reds') tf.add_gaussian(2, 1, [1.0, 0.8, 0.0, 1.0]) tf.add_gaussian(3, 2, [1.0, 0.5, 0.0, 1.0]) tf.add_gaussian(5, 3, [1.0, 0.0, 0.0, 1.0]) tf.add_gaussian(10, 5, [1.0, 0.0, 0.0, 0.5]) tf.map_to_colormap(10, 30, colormap=red, scale=1) center = ytcube.dataset.domain_dimensions / 2. cam = ytcube.dataset.h.camera(center, camera_angle, scale, size, tf, north_vector=north_vector, fields='flux') im = cam.snapshot() #images = [im] if movie: pb = ProgressBar(nframes * 2 + 30) for ii, im in enumerate( cam.rotation(2 * np.pi, nframes / 3, rot_vector=rot_vector1)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir, "%04i.png" % (ii))), rescale=False) pb.update(ii) for jj, im in enumerate( cam.rotation(2 * np.pi, nframes / 3, rot_vector=rot_vector2)): #images.append(im) im.write_png(paths.mpath( os.path.join(outdir, "%04i.png" % (ii + jj))), rescale=False) pb.update(ii + jj) for kk, im in enumerate( cam.rotation(2 * np.pi, nframes / 3, rot_vector=rot_vector3)): #images.append(im) im.write_png(paths.mpath( os.path.join(outdir, "%04i.png" % (ii + jj + kk))), rescale=False) pb.update(ii + jj + kk) TheBrick = ytcube.world2yt([0.253, 0.016, 35]) for LL, snapshot in enumerate(cam.move_to(TheBrick, 15)): #images.append(snapshot) snapshot.write_png(paths.mpath( os.path.join(outdir, '%04i.png' % (ii + jj + kk + LL))), rescale=False) pb.update(ii + jj + kk + LL) for mm, snapshot in enumerate(cam.zoomin(5, 15)): #images.append(snapshot) snapshot.write_png(paths.mpath( os.path.join(outdir, '%04i.png' % (ii + jj + kk + LL + mm))), rescale=False) pb.update(ii + jj + kk + LL + mm) for nn, im in enumerate( cam.rotation(2 * np.pi, nframes / 3, rot_vector=rot_vector1)): #images.append(im) im.write_png(paths.mpath( os.path.join(outdir, "%04i.png" % (ii + jj + kk + LL + mm + nn))), rescale=False) pb.update(ii + jj + kk + LL + mm + nn) for oo, im in enumerate( cam.rotation(2 * np.pi, nframes / 3, rot_vector=rot_vector2)): #images.append(im) im.write_png(paths.mpath( os.path.join(outdir, "%04i.png" % (ii + jj + kk + LL + mm + nn + oo))), rescale=False) pb.update(ii + jj + kk + LL + mm + nn + oo) for pp, im in enumerate( cam.rotation(2 * np.pi, nframes / 3, rot_vector=rot_vector3)): #images.append(im) im.write_png(paths.mpath( os.path.join( outdir, "%04i.png" % (ii + jj + kk + LL + mm + nn + oo + pp))), rescale=False) pb.update(ii + jj + kk + LL + mm + nn + oo + pp) #save_images(images, paths.mpath(outdir)) pipe = make_movie(paths.mpath(outdir)) return images else: return im
vel_slopes.append(vel_slope.slope) dens_spec = threeD_pspec(density.value) dens_slope = linregress(np.log10(dens_spec[0][:-1]), np.log10(dens_spec[1][:-1])) dens_slopes.append(dens_slope.slope) cube_hdu = make_ppv(velocity, density, vel_disp=np.std(velocity), T=T, threads=4, verbose=True, chan_width=dv_eff / 2., v_min=-60 * u.km / u.s, v_max=60 * u.km / u.s, max_chan=2000) filename = "fBM_density_{0:.2f}_velocity_{1:.2f}_rep_{2}_size_{3}.fits"\ .format(np.abs(dens), np.abs(vel), i, cube_size) cube_hdu.writeto(osjoin(out_dir, filename), overwrite=True) bar.update() filename = "fBM_3D_velocity_slopes_size_{0}.npy"\ .format(cube_size) np.save(osjoin(out_dir, filename), np.array(vel_slopes)) filename = "fBM_3D_density_slopes_size_{0}.npy"\ .format(cube_size) np.save(osjoin(out_dir, filename), np.array(dens_slopes))
def match(*args, verbose=True): """ Find sources that match up between any number of dendrocat objects. Parameters ---------- obj1 : rsprocess.RadioSource object or rsprocess.MasterCatalog object A catalog with which to compare radio sources. obj2 : rsprocess.RadioSource object or rsprocess.MasterCatalog object A catalog with which to compare radio sources. Returns ---------- astropy.table.Table object """ from .mastercatalog import MasterCatalog current_arg = args[0] for k in range(len(args) - 1): obj1 = current_arg obj2 = args[k + 1] all_colnames = set(obj1.catalog.colnames + obj2.catalog.colnames) stack = vstack([obj1.catalog, obj2.catalog]) all_colnames.add('_index') try: stack.add_column(Column(range(len(stack)), name='_index')) except ValueError: stack['_index'] = range(len(stack)) stack = stack[sorted(list(all_colnames))] rejected = np.where(stack['rejected'] == 1)[0] if verbose: print('Combining matches') pb = ProgressBar(len(stack) - len(rejected)) i = 0 while True: if i == len(stack) - 1: break if i in rejected: i += 1 continue teststar = stack[i] delta_p = deepcopy(stack[stack['rejected'] == 0]['_idx', '_index', 'x_cen', 'y_cen']) delta_p.remove_rows( np.where(delta_p['_index'] == teststar['_index'])[0]) delta_p['x_cen'] = np.abs(delta_p['x_cen'] - teststar['x_cen']) delta_p['y_cen'] = np.abs(delta_p['y_cen'] - teststar['y_cen']) delta_p.sort('x_cen') threshold = 1e-5 found_match = False dist_col = MaskedColumn(length=len(delta_p), name='dist', mask=True) for j in range(10): dist_col[j] = np.sqrt(delta_p[j]['x_cen']**2. + delta_p[j]['y_cen']**2) if dist_col[j] <= threshold: found_match = True delta_p.add_column(dist_col) delta_p.sort('dist') if found_match: match_index = np.where(stack['_index'] == delta_p[0]['_index']) match = deepcopy(stack[match_index]) stack.remove_row(match_index[0][0]) # Find the common bounding ellipse new_x_cen = np.average([match['x_cen'], teststar['x_cen']]) new_y_cen = np.average([match['y_cen'], teststar['y_cen']]) # Find new ellipse properties new_maj, new_min, new_pa = commonbeam( float(match['major_fwhm']), float(match['minor_fwhm']), float(match['position_angle']), float(teststar['major_fwhm']), float(teststar['minor_fwhm']), float(teststar['position_angle'])) # Replace properties of test star stack[i]['x_cen'] = new_x_cen stack[i]['y_cen'] = new_y_cen stack[i]['major_fwhm'] = new_maj.value stack[i]['minor_fwhm'] = new_min.value stack[i]['position_angle'] = new_pa.value # Replace masked data with available values from the match for k, masked in enumerate(stack.mask[i]): colname = stack.colnames[k] if masked: stack[i][colname] = match[colname] i += 1 if verbose: pb.update() # Fill masked detection column fields with 'False' for colname in stack.colnames: if 'detected' in colname: stack[colname].fill_value = 0 stack['_index'] = range(len(stack)) current_arg = MasterCatalog(obj1, obj2, catalog=stack) return current_arg
def pz_to_catalog(pz, zgrid, catalog, verbose=True): output = Table() pri_peakz = np.zeros_like(catalog['z_spec']) pri_upper = np.zeros_like(catalog['z_spec']) pri_lower = np.zeros_like(catalog['z_spec']) pri_area = np.zeros_like(catalog['z_spec']) pri_peakz.name = 'z1_median' pri_upper.name = 'z1_max' pri_lower.name = 'z1_min' pri_area.name = 'z1_area' pri_peakz.format = '%.4f' pri_upper.format = '%.4f' pri_lower.format = '%.4f' pri_area.format = '%.3f' sec_peakz = np.zeros_like(catalog['z_spec']) sec_upper = np.zeros_like(catalog['z_spec']) sec_lower = np.zeros_like(catalog['z_spec']) sec_area = np.zeros_like(catalog['z_spec']) sec_peakz.name = 'z2_median' sec_upper.name = 'z2_max' sec_lower.name = 'z2_min' sec_area.name = 'z2_area' sec_peakz.format = '%.4f' sec_upper.format = '%.4f' sec_lower.format = '%.4f' sec_area.format = '%.3f' if verbose: bar = ProgressBar(len(pz)) for i, pzi in enumerate(pz): peaks, l80s, u80s, areas = get_peak_z(pzi, zgrid) peaks = np.array(peaks, ndmin=1) l80s = np.array(l80s, ndmin=1) u80s = np.array(u80s, ndmin=1) areas = np.array(areas, ndmin=1) if np.isnan(peaks[0]): pri_peakz[i] = -99. else: pri_peakz[i] = peaks[0] pri_upper[i] = u80s[0] pri_lower[i] = l80s[0] pri_area[i] = areas[0] if len(peaks) > 1: sec_peakz[i] = peaks[1] sec_upper[i] = u80s[1] sec_lower[i] = l80s[1] sec_area[i] = areas[1] else: sec_peakz[i] = -99. sec_upper[i] = -99. sec_lower[i] = -99. sec_area[i] = -99. if verbose: bar.update() output.add_column(catalog['id']) output.add_column(pri_peakz) output.add_column(pri_lower) output.add_column(pri_upper) output.add_column(pri_area) output.add_column(sec_peakz) output.add_column(sec_lower) output.add_column(sec_upper) output.add_column(sec_area) return output
def fourier_combine_cubes(cube1, cube2, highresextnum=0, highresscalefactor=1.0, lowresscalefactor=1.0, lowresfwhm=1*u.arcmin, return_regridded_cube2=False, return_hdu=False, ): """ Fourier combine two data cubes Parameters ---------- cube1 : SpectralCube highresfitsfile : str The high-resolution FITS file cube2 : SpectralCube lowresfitsfile : str The low-resolution (single-dish) FITS file highresextnum : int The extension number to use from the high-res FITS file highresscalefactor : float lowresscalefactor : float A factor to multiply the high- or low-resolution data by to match the low- or high-resolution data lowresfwhm : `astropy.units.Quantity` The full-width-half-max of the single-dish (low-resolution) beam; or the scale at which you want to try to match the low/high resolution data return_hdu : bool Return an HDU instead of just a cube. It will contain two image planes, one for the real and one for the imaginary data. return_regridded_cube2 : bool Return the 2nd cube regridded into the pixel space of the first? """ if isinstance(cube1, str): cube1 = SpectralCube.read(cube1) if isinstance(cube2, str): cube2 = SpectralCube.read(cube2) #cube1 = spectral_cube.io.fits.load_fits_cube(highresfitsfile, # hdu=highresextnum) im1 = cube1._data # want the raw data for this hd1 = cube1.header assert hd1['NAXIS'] == im1.ndim == 3 w1 = cube1.wcs pixscale = np.abs(w1.wcs.get_cdelt()[0]) # REPLACE EVENTUALLY... cube2 = cube2.to(cube1.unit) assert cube1.unit == cube2.unit, 'Cubes must have same or equivalent unit' assert cube1.unit.is_equivalent(u.Jy/u.beam) or cube1.unit.is_equivalent(u.K), "Cubes must have brightness units." #f2 = regrid_fits_cube(lowresfitsfile, hd1) f2 = regrid_cube_hdu(cube2.hdu, hd1) w2 = wcs.WCS(f2.header) nax1,nax2,nax3 = (hd1['NAXIS1'], hd1['NAXIS2'], hd1['NAXIS3']) dcube1 = im1 * highresscalefactor dcube2 = f2.data * lowresscalefactor outcube = np.empty_like(dcube1) xgrid,ygrid = (np.indices([nax2,nax1])-np.array([(nax2-1.)/2,(nax1-1.)/2.])[:,None,None]) fwhm = np.sqrt(8*np.log(2)) # sigma in pixels sigma = ((lowresfwhm/fwhm/(pixscale*u.deg)).decompose().value) #sigma_fftspace = (1/(4*np.pi**2*sigma**2))**0.5 sigma_fftspace = (2*np.pi*sigma)**-1 log.debug('sigma = {0}, sigma_fftspace={1}'.format(sigma, sigma_fftspace)) kernel = np.fft.fftshift(np.exp(-(xgrid**2+ygrid**2)/(2*sigma**2))) # convert the kernel, which is just a gaussian in image space, # to its corresponding kernel in fourier space kfft = np.abs(np.fft.fft2(kernel)) # should be mostly real # normalize the kernel kfft/=kfft.max() ikfft = 1-kfft pb = ProgressBar(dcube1.shape[0]) for ii,(im1,im2) in enumerate(zip(dcube1, dcube2)): fft1 = np.fft.fft2(np.nan_to_num(im1)) fft2 = np.fft.fft2(np.nan_to_num(im2)) fftsum = kfft*fft2 + ikfft*fft1 combo = np.fft.ifft2(fftsum) outcube[ii,:,:] = combo.real pb.update(ii+1) if return_regridded_cube2: return outcube, f2 elif return_hdu: return fits.PrimaryHDU(data=outcube, header=w1.to_header()) else: return outcube
def _convolve_model_dir_1(model_dir, filters, overwrite=False): for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + "/convolved"): os.mkdir(model_dir + "/convolved") # Find all SED files to convolve sed_files = ( glob.glob(model_dir + "/seds/*.fits.gz") + glob.glob(model_dir + "/seds/*/*.fits.gz") + glob.glob(model_dir + "/seds/*.fits") + glob.glob(model_dir + "/seds/*/*.fits") ) par_table = load_parameter_table(model_dir) if len(sed_files) == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(len(sed_files), model_dir)) # Find out apertures first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures # Set up convolved fluxes fluxes = [ ConvolvedFluxes( model_names=np.zeros(len(sed_files), dtype="U30" if six.PY3 else "S30"), apertures=apertures, initialize_arrays=True, ) for i in range(len(filters)) ] # Set up list of binned filters binned_filters = [] binned_nu = None # Loop over SEDs b = ProgressBar(len(sed_files)) for im, sed_file in enumerate(sed_files): log.debug("Convolving {0}".format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order="nu") # Check if filters need to be re-binned try: assert binned_nu is not None np.testing.assert_array_almost_equal_nulp(s.nu.value, binned_nu.value, 100) except AssertionError: log.info("Rebinning filters") binned_filters = [f.rebin(s.nu) for f in filters] binned_nu = s.nu b.update() # Convolve for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = apertures fluxes[i].model_names[im] = s.name if n_ap == 1: fluxes[i].flux[im] = np.sum(s.flux * f.response) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2)) else: fluxes[i].flux[im, :] = np.sum(s.flux * f.response, axis=1) fluxes[i].error[im] = np.sqrt(np.sum((s.error * f.response) ** 2, axis=1)) for i, f in enumerate(binned_filters): fluxes[i].sort_to_match(par_table["MODEL_NAME"]) fluxes[i].write(model_dir + "/convolved/" + f.name + ".fits", overwrite=overwrite)
def add_data_to_cube(cubefilename, data=None, filename=None, fileheader=None, flatheader='header.txt', cubeheader='cubeheader.txt', nhits=None, smoothto=1, baselineorder=5, velocityrange=None, excludefitrange=None, noisecut=np.inf, do_runscript=False, linefreq=None, allow_smooth=True, data_iterator=data_iterator, coord_iterator=coord_iterator, velo_iterator=velo_iterator, progressbar=False, coordsys='galactic', datalength=None, velocity_offset=0.0, negative_mean_cut=None, add_with_kernel=False, kernel_fwhm=None, fsw=False, kernel_function=Gaussian2DKernel, diagnostic_plot_name=None, chmod=False, continuum_prefix=None, debug_breakpoint=False, default_unit=u.km/u.s, make_continuum=True, weightspec=None, varweight=False): """ Given a .fits file that contains a binary table of spectra (e.g., as you would get from the GBT mapping "pipeline" or the reduce_map.pro aoidl file provided by Adam Ginsburg), adds each spectrum into the cubefile. velocity_offset : 0.0 Amount to add to the velocity vector before adding it to the cube (useful for FSW observations) weightspec : np.ndarray A spectrum with the same size as the input arrays but containing the relative weights of the data """ #if not default_unit.is_equivalent(u.km/u.s): # raise TypeError("Default unit is not a velocity equivalent.") if type(nhits) is str: log.debug("Loading nhits from %s" % nhits) nhits = pyfits.getdata(nhits) elif type(nhits) is not np.ndarray: raise TypeError("nhits must be a .fits file or an ndarray, but it is ",type(nhits)) naxis2,naxis1 = nhits.shape if velocity_offset and not fsw: raise ValueError("Using a velocity offset, but obs type is not " "frequency switched; this is almost certainly wrong, " "but if there's a case for it I'll remove this.") if not hasattr(velocity_offset,'unit'): velocity_offset = velocity_offset*default_unit contimage = np.zeros_like(nhits) nhits_once = np.zeros_like(nhits) log.debug("Loading data cube {0}".format(cubefilename)) t0 = time.time() # rescale image to weight by number of observations image = pyfits.getdata(cubefilename)*nhits log.debug(" ".join(("nhits statistics: mean, std, nzeros, size",str(nhits.mean()),str(nhits.std()),str(np.sum(nhits==0)), str(nhits.size)))) log.debug(" ".join(("Image statistics: mean, std, nzeros, size",str(image.mean()),str(image.std()),str(np.sum(image==0)), str(image.size), str(np.sum(np.isnan(image)))))) log.debug(" ".join(("nhits shape: ",str(nhits.shape)))) # default is to set empty pixels to NAN; have to set them # back to zero image[image!=image] = 0.0 header = pyfits.getheader(cubefilename) # debug print "Cube shape: ",image.shape," naxis3: ",header.get('NAXIS3')," nhits shape: ",nhits.shape log.debug("".join(("Image statistics: mean, std, nzeros, size",str(image.mean()),str(image.std()),str(np.sum(image==0)), str(image.size)))) flathead = get_header(flatheader) naxis3 = image.shape[0] wcs = pywcs.WCS(flathead) cwcs = pywcs.WCS(header) vwcs = cwcs.sub([pywcs.WCSSUB_SPECTRAL]) vunit = u.Unit(vwcs.wcs.cunit[vwcs.wcs.spec]) cubevelo = vwcs.wcs_pix2world(np.arange(naxis3),0)[0] * vunit cd3 = vwcs.wcs.cdelt[vwcs.wcs.spec] * vunit if not vunit.is_equivalent(default_unit): raise ValueError("The units of the cube and the velocity axis are " "possibly not equivalent. Change default_unit to " "the appropriate unit (probably {0})".format(vunit)) if add_with_kernel: if wcs.wcs.has_cd(): cd = np.abs(wcs.wcs.cd[1,1]) else: cd = np.abs(wcs.wcs.cdelt[1]) # Alternative implementation; may not work for .cd? #cd = np.abs(np.prod((wcs.wcs.get_cdelt() * wcs.wcs.get_pc().diagonal())))**0.5 if velocityrange is not None: if hasattr(velocityrange, 'unit'): v1,v4 = velocityrange else: v1,v4 = velocityrange * default_unit ind1 = np.argmin(np.abs(np.floor(v1-cubevelo))) ind2 = np.argmin(np.abs(np.ceil(v4-cubevelo)))+1 # stupid hack. REALLY stupid hack. Don't crop. if np.abs(ind2-image.shape[0]) < 5: ind2 = image.shape[0] if np.abs(ind1) < 5: ind1 = 0 #print "Velo match for v1,v4 = %f,%f: %f,%f" % (v1,v4,cubevelo[ind1],cubevelo[ind2]) # print "Updating CRPIX3 from %i to %i. Cropping to indices %i,%i" % (header.get('CRPIX3'),header.get('CRPIX3')-ind1,ind1,ind2) # I think this could be disastrous: cubevelo is already set, but now we're changing how it's set in the header! # I don't think there's any reason to have this in the first place # header.set('CRPIX3',header.get('CRPIX3')-ind1) # reset v1,v4 to the points we just selected v1 = cubevelo[ind1] v4 = cubevelo[ind2-1] else: ind1=0 ind2 = image.shape[0] v1,v4 = min(cubevelo),max(cubevelo) # debug print "Cube has %i v-axis pixels from %f to %f. Crop range is %f to %f" % (naxis3,cubevelo.min(),cubevelo.max(),v1,v4) #if abs(cdelt) < abs(cd3): # print "Spectra have CD=%0.2f, cube has CD=%0.2f. Will smooth & interpolate." % (cdelt,cd3) # Disable progressbar if debug-logging is enabled (they clash) if progressbar and 'ProgressBar' in globals() and log.level > 10: if datalength is None: pb = ProgressBar(len(data)) else: pb = ProgressBar(datalength) else: progressbar = False skipped = [] for spectrum,pos,velo in zip(data_iterator(data,fsw=fsw), coord_iterator(data,coordsys_out=coordsys), velo_iterator(data,linefreq=linefreq)): if log.level <= 10: t1 = time.time() if not hasattr(velo,'unit'): velo = velo * default_unit glon,glat = pos cdelt = velo[1]-velo[0] if cdelt < 0: # for interpolation, require increasing X axis spectrum = spectrum[::-1] velo = velo[::-1] if log.level < 5: log.debug("Reversed spectral axis... ") if (velo.max() < cubevelo.min() or velo.min() > cubevelo.max()): raise ValueError("Data out of range.") if progressbar and log.level > 10: pb.update() velo += velocity_offset if glon != 0 and glat != 0: x,y = wcs.wcs_world2pix(glon,glat,0) if np.isnan(x) or np.isnan(y): log.warn("".join(("Skipping NaN point {0}, {1} ...".format(glon,glat)))) continue if log.level < 10: log.debug("".join(("At point {0},{1} ...".format(glon,glat),))) if abs(cdelt) < abs(cd3) and allow_smooth: # need to smooth before interpolating to preserve signal kernwidth = abs(cd3/cdelt/2.35).decompose().value if kernwidth > 2 and kernwidth < 10: xr = kernwidth*5 npx = np.ceil(xr*2 + 1) elif kernwidth > 10: raise ValueError('Too much smoothing') else: xr = 5 npx = 11 #kernel = np.exp(-(np.linspace(-xr,xr,npx)**2)/(2.0*kernwidth**2)) #kernel /= kernel.sum() kernel = Gaussian1DKernel(stddev=kernwidth, x_size=npx) smspec = np.convolve(spectrum,kernel,mode='same') datavect = np.interp(cubevelo.to(default_unit).value, velo.to(default_unit).value, smspec) else: datavect = np.interp(cubevelo.to(default_unit).value, velo.to(default_unit).value, spectrum) OK = (datavect[ind1:ind2] == datavect[ind1:ind2]) if excludefitrange is None: include = OK else: # Exclude certain regions (e.g., the spectral lines) when computing the noise include = OK.copy() if not hasattr(excludefitrange,'unit'): excludefitrange = excludefitrange * default_unit # Convert velocities to indices exclude_inds = [np.argmin(np.abs(np.floor(v-cubevelo))) for v in excludefitrange] # Loop through exclude_inds pairwise for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]): # Do not include the excluded regions include[i1:i2] = False if include.sum() == 0: raise ValueError("All data excluded.") noiseestimate = datavect[ind1:ind2][include].std() contestimate = datavect[ind1:ind2][include].mean() if noiseestimate > noisecut: log.info("Skipped a data point at %f,%f in file %s because it had excessive noise %f" % (x,y,filename,noiseestimate)) skipped.append(True) continue elif negative_mean_cut is not None and contestimate < negative_mean_cut: log.info("Skipped a data point at %f,%f in file %s because it had negative continuum %f" % (x,y,filename,contestimate)) skipped.append(True) continue elif OK.sum() == 0: log.info("Skipped a data point at %f,%f in file %s because it had NANs" % (x,y,filename)) skipped.append(True) continue elif OK.sum()/float(abs(ind2-ind1)) < 0.5: log.info("Skipped a data point at %f,%f in file %s because it had %i NANs" % (x,y,filename,np.isnan(datavect[ind1:ind2]).sum())) skipped.append(True) continue if log.level < 10: log.debug("did not skip...") if varweight: weight = 1./noiseestimate**2 else: weight = 1. if weightspec is None: wspec = weight else: wspec = weight * weightspec if 0 < int(np.round(x)) < naxis1 and 0 < int(np.round(y)) < naxis2: if add_with_kernel: fwhm = np.sqrt(8*np.log(2)) kernel_size = kd = int(np.ceil(kernel_fwhm/fwhm/cd * 5)) if kernel_size < 5: kernel_size = kd = 5 if kernel_size % 2 == 0: kernel_size = kd = kernel_size+1 if kernel_size > 100: raise ValueError("Huge kernel - are you sure?") kernel_middle = mid = (kd-1)/2. xinds,yinds = (np.mgrid[:kd,:kd]-mid+np.array([np.round(x),np.round(y)])[:,None,None]).astype('int') # This kernel is NOT centered, and that's the bloody point. # (I made a very stupid error and used Gaussian2DKernel, # which is strictly centered, in a previous version) kernel2d = np.exp(-((xinds-x)**2+(yinds-y)**2)/(2*(kernel_fwhm/fwhm/cd)**2)) dim1 = ind2-ind1 vect_to_add = np.outer(datavect[ind1:ind2],kernel2d).reshape([dim1,kd,kd]) vect_to_add[True-OK] = 0 # need to slice out edges if yinds.max() >= naxis2 or yinds.min() < 0: yok = (yinds[0,:] < naxis2) & (yinds[0,:] >= 0) xinds,yinds = xinds[:,yok],yinds[:,yok] vect_to_add = vect_to_add[:,:,yok] kernel2d = kernel2d[:,yok] if xinds.max() >= naxis1 or xinds.min() < 0: xok = (xinds[:,0] < naxis1) & (xinds[:,0] >= 0) xinds,yinds = xinds[xok,:],yinds[xok,:] vect_to_add = vect_to_add[:,xok,:] kernel2d = kernel2d[xok,:] image[ind1:ind2,yinds,xinds] += vect_to_add*wspec # NaN spectral bins are not appropriately downweighted... but they shouldn't exist anyway... nhits[yinds,xinds] += kernel2d*weight contimage[yinds,xinds] += kernel2d * contestimate*weight nhits_once[yinds,xinds] += kernel2d*weight else: image[ind1:ind2,int(np.round(y)),int(np.round(x))][OK] += datavect[ind1:ind2][OK]*weight nhits[int(np.round(y)),int(np.round(x))] += weight contimage[int(np.round(y)),int(np.round(x))] += contestimate*weight nhits_once[int(np.round(y)),int(np.round(x))] += weight if log.level < 10: log.debug("Z-axis indices are %i,%i..." % (ind1,ind2,)) log.debug("Added a data point at %i,%i" % (int(np.round(x)),int(np.round(y)))) skipped.append(False) else: skipped.append(True) log.info("Skipped a data point at x,y=%f,%f " "lon,lat=%f,%f in file %s because " "it's out of the grid" % (x,y,glon,glat,filename)) if debug_breakpoint: import ipdb ipdb.set_trace() if log.level <= 10: dt = time.time() - t1 log.debug("Completed x,y={x:4.0f},{y:4.0f}" " ({x:6.2f},{y:6.2f}) in {dt:6.2g}s".format(x=float(x), y=float(y), dt=dt)) log.info("Completed 'add_data' loop for" " {0} in {1}s".format(cubefilename, time.time()-t0)) if excludefitrange is not None: # this block redefining "include" is used for diagnostics (optional) ind1a = np.argmin(np.abs(np.floor(v1-velo))) ind2a = np.argmin(np.abs(np.ceil(v4-velo)))+1 dname = 'DATA' if 'DATA' in data.dtype.names else 'SPECTRA' OK = (data[dname][0,:]==data[dname][0,:]) OK[:ind1a] = False OK[ind2a:] = False include = OK # Convert velocities to indices exclude_inds = [np.argmin(np.abs(np.floor(v-velo))) for v in excludefitrange] # Loop through exclude_inds pairwise for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]): # Do not include the excluded regions include[i1:i2] = False if include.sum() == 0: raise ValueError("All data excluded.") else: dname = 'DATA' if 'DATA' in data.dtype.names else 'SPECTRA' include = slice(None) if diagnostic_plot_name: from mpl_plot_templates import imdiagnostics pylab.clf() dd = data[dname][:,include] imdiagnostics(dd,axis=pylab.gca()) pylab.savefig(diagnostic_plot_name, bbox_inches='tight') # Save a copy with the bad stuff flagged out; this should tell whether flagging worked skipped = np.array(skipped,dtype='bool') dd[skipped,:] = -999 maskdata = np.ma.masked_equal(dd,-999) pylab.clf() imdiagnostics(maskdata, axis=pylab.gca()) dpn_pre,dpn_suf = os.path.splitext(diagnostic_plot_name) dpn_flagged = dpn_pre+"_flagged"+dpn_suf pylab.savefig(dpn_flagged, bbox_inches='tight') log.info("Saved diagnostic plot %s and %s" % (diagnostic_plot_name,dpn_flagged)) log.debug("nhits statistics: mean, std, nzeros, size {0} {1} {2} {3}".format(nhits.mean(),nhits.std(),np.sum(nhits==0), nhits.size)) log.debug("Image statistics: mean, std, nzeros, size {0} {1} {2} {3}".format(image.mean(),image.std(),np.sum(image==0), image.size)) imav = image/nhits if log.level <= 10: nnan = np.count_nonzero(np.isnan(imav)) log.debug("imav statistics: mean, std, nzeros, size, nnan, ngood: {0} {1} {2} {3} {4} {5}".format(imav.mean(),imav.std(),np.sum(imav==0), imav.size, nnan, imav.size-nnan)) log.debug("imav shape: {0}".format(imav.shape)) subcube = imav[ind1:ind2,:,:] if log.level <= 10: nnan = np.sum(np.isnan(subcube)) print("subcube statistics: mean, std, nzeros, size, nnan, ngood:",np.nansum(subcube)/subcube.size,np.std(subcube[subcube==subcube]),np.sum(subcube==0), subcube.size, nnan, subcube.size-nnan) print("subcube shape: ",subcube.shape) H = header.copy() if fileheader is not None: for k,v in fileheader.items(): if 'RESTFRQ' in k or 'RESTFREQ' in k: header.set(k,v) #if k[0] == 'C' and '1' in k and k[-1] != '1': # header.set(k.replace('1','3'), v) moreH = get_header(cubeheader) for k,v in H.items(): header.set(k,v) for k,v in moreH.items(): header.set(k,v) HDU = pyfits.PrimaryHDU(data=subcube,header=header) HDU.writeto(cubefilename,clobber=True,output_verify='fix') outpre = cubefilename.replace(".fits","") include = np.ones(imav.shape[0],dtype='bool') if excludefitrange is not None: # this block redifining "include" is used for continuum ind1a = np.argmin(np.abs(np.floor(v1-cubevelo))) ind2a = np.argmin(np.abs(np.ceil(v4-cubevelo)))+1 # Convert velocities to indices exclude_inds = [np.argmin(np.abs(np.floor(v-cubevelo))) for v in excludefitrange] # Loop through exclude_inds pairwise for (i1,i2) in zip(exclude_inds[:-1:2],exclude_inds[1::2]): # Do not include the excluded regions include[i1:i2] = False if include.sum() == 0: raise ValueError("All data excluded.") HDU2 = pyfits.PrimaryHDU(data=nhits,header=flathead) HDU2.writeto(outpre+"_nhits.fits",clobber=True,output_verify='fix') #OKCube = (imav==imav) #contmap = np.nansum(imav[naxis3*0.1:naxis3*0.9,:,:],axis=0) / OKCube.sum(axis=0) if make_continuum: contmap = np.nansum(imav[include,:,:],axis=0) / include.sum() HDU2 = pyfits.PrimaryHDU(data=contmap,header=flathead) HDU2.writeto(outpre+"_continuum.fits",clobber=True,output_verify='fix') if continuum_prefix is not None: # Solo continuum image (just this obs set) HDU2.data = contimage / nhits_once HDU2.writeto(continuum_prefix+"_continuum.fits",clobber=True,output_verify='fix') HDU2.data = nhits_once HDU2.writeto(continuum_prefix+"_nhits.fits",clobber=True,output_verify='fix') log.info("Writing script file {0}".format(outpre+"_starlink.sh")) scriptfile = open(outpre+"_starlink.sh",'w') outpath,outfn = os.path.split(cubefilename) outpath,pre = os.path.split(outpre) print(("#!/bin/bash"), file=scriptfile) if outpath != '': print(('cd %s' % outpath), file=scriptfile) print(('. /star/etc/profile'), file=scriptfile) print(('kappa > /dev/null'), file=scriptfile) print(('convert > /dev/null'), file=scriptfile) print(('fits2ndf %s %s' % (outfn,outfn.replace(".fits",".sdf"))), file=scriptfile) if excludefitrange is not None: v2v3 = "" for v2,v3 in zip(excludefitrange[::2],excludefitrange[1::2]): v2v3 += "%0.2f %0.2f " % (v2.to(default_unit).value,v3.to(default_unit).value) print(('mfittrend %s ranges=\\\"%0.2f %s %0.2f\\\" order=%i axis=3 out=%s' % (outfn.replace(".fits",".sdf"),v1.to(default_unit).value,v2v3,v4.to(default_unit).value,baselineorder,outfn.replace(".fits","_baseline.sdf"))), file=scriptfile) else: print(('mfittrend %s ranges=\\\"%0.2f %0.2f\\\" order=%i axis=3 out=%s' % (outfn.replace(".fits",".sdf"),v1.to(default_unit).value,v4.to(default_unit).value,baselineorder,outfn.replace(".fits","_baseline.sdf"))), file=scriptfile) print(('sub %s %s %s' % (outfn.replace(".fits",".sdf"),outfn.replace(".fits","_baseline.sdf"),outfn.replace(".fits","_sub.sdf"))), file=scriptfile) print(('sqorst %s_sub mode=pixelscale axis=3 pixscale=%i out=%s_vrebin' % (pre,smoothto,pre)), file=scriptfile) print(('gausmooth %s_vrebin fwhm=1.0 axes=[1,2] out=%s_smooth' % (pre,pre)), file=scriptfile) print(('#collapse %s estimator=mean axis="VRAD" low=-400 high=500 out=%s_continuum' % (pre,pre)), file=scriptfile) print(('rm %s_sub.fits' % (pre)), file=scriptfile) print(('ndf2fits %s_sub %s_sub.fits' % (pre,pre)), file=scriptfile) print(('rm %s_smooth.fits' % (pre)), file=scriptfile) print(('ndf2fits %s_smooth %s_smooth.fits' % (pre,pre)), file=scriptfile) print(("# Fix STARLINK's failure to respect header keywords."), file=scriptfile) print(('sethead %s_smooth.fits RESTFRQ=`gethead RESTFRQ %s.fits`' % (pre,pre)), file=scriptfile) print(('rm %s_baseline.sdf' % (pre)), file=scriptfile) print(('rm %s_smooth.sdf' % (pre)), file=scriptfile) print(('rm %s_sub.sdf' % (pre)), file=scriptfile) print(('rm %s_vrebin.sdf' % (pre)), file=scriptfile) print(('rm %s.sdf' % (pre)), file=scriptfile) scriptfile.close() if chmod: scriptfilename = (outpre+"_starlink.sh").replace(" ","") #subprocess.call("chmod +x {0}".format(scriptfilename), shell=True) st = os.stat(scriptfilename) os.chmod(scriptfilename, st.st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH | stat.S_IXUSR) if do_runscript: runscript(outpre) _fix_ms_kms_file(outpre+"_sub.fits") _fix_ms_kms_file(outpre+"_smooth.fits") if log.level <= 20: log.info("Completed {0} in {1}s".format(pre, time.time()-t0))
cube = SpectralCube.read(interferometer_fn).with_spectral_unit(u.km/u.s, velocity_convention='radio') cube.beam_threshold = 100 # try to avoid contamination... won't work universally; need to examine # individual cubes and have this as a parameter #med = cube.spectral_slab(90*u.km/u.s, 160*u.km/u.s).median(axis=0).value med = cube.spectral_slab(velocity_ranges[species][0]*u.km/u.s, velocity_ranges[species][1]*u.km/u.s).median(axis=0).value os.system('cp {0} {1}'.format(interferometer_fn, medsubfn)) fh = fits.open(medsubfn, mode='update') log.info("Median subtracting") pb = ProgressBar(len(fh[0].data)) for ii,imslice in enumerate(fh[0].data): fh[0].data[ii] = imslice - med fh.flush() pb.update() fh.close() cube = SpectralCube.read(medsubfn).with_spectral_unit(u.GHz) if hasattr(cube, 'beam'): jtok = cube.beam.jtok(cube.spectral_axis).value print("Jansky/beam -> Kelvin factor = {0}".format(jtok)) else: jtok = np.array([bm.jtok(x).value for bm,x in zip(cube.beams, cube.spectral_axis)]) print("Median Jansky/beam -> Kelvin factor = {0}".format(np.median(jtok))) minghz, maxghz = cube.spectral_extrema OK = False
def convolve_model_dir_monochromatic(model_dir, overwrite=False, max_ram=8, wav_min=-np.inf * u.micron, wav_max=np.inf * u.micron): """ Convolve all the model SEDs in a model directory Parameters ---------- model_dir : str The path to the model directory overwrite : bool, optional Whether to overwrite the output files max_ram : float, optional The maximum amount of RAM that can be used (in Gb) wav_min : float, optional The minimum wavelength to consider. Only wavelengths above this value will be output. wav_max : float, optional The maximum wavelength to consider. Only wavelengths below this value will be output. """ modpar = parfile.read(os.path.join(model_dir, 'models.conf'), 'conf') if modpar.get('version', 1) > 1: raise ValueError( "monochromatic filters are no longer used for new-style model directories" ) # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_files = sorted( glob.glob(model_dir + '/seds/*.fits.gz') + glob.glob(model_dir + '/seds/*/*.fits.gz') + glob.glob(model_dir + '/seds/*.fits') + glob.glob(model_dir + '/seds/*/*.fits')) par_table = load_parameter_table(model_dir) # Find number of models n_models = len(sed_files) if n_models == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(n_models, model_dir)) # Find out apertures and wavelengths first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures n_wav = first_sed.n_wav wavelengths = first_sed.wav # For model grids that are very large, it is not possible to compute all # fluxes in one go, so we need to process in chunks in wavelength space. chunk_size = min( n_wav, int(np.floor(max_ram * 1024.**3 / (4. * 2. * n_models * n_ap)))) if chunk_size == n_wav: log.info("Producing all monochromatic files in one go") else: log.info("Producing monochromatic files in chunks of {0}".format( chunk_size)) filters = Table() filters['wav'] = wavelengths filters['filter'] = np.zeros(wavelengths.shape, dtype='S10') # Figure out range of wavelength indices to use # (wavelengths array is sorted in reverse order) jlo = n_wav - 1 - (wavelengths[::-1].searchsorted(wav_max) - 1) jhi = n_wav - 1 - wavelengths[::-1].searchsorted(wav_min) chunk_size = min(chunk_size, jhi - jlo + 1) # Loop over wavelength chunks for jmin in range(jlo, jhi, chunk_size): # Find upper wavelength to compute jmax = min(jmin + chunk_size - 1, jhi) log.info('Processing wavelengths {0} to {1}'.format(jmin, jmax)) # Set up convolved fluxes fluxes = [ ConvolvedFluxes(model_names=np.zeros( n_models, dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(chunk_size) ] b = ProgressBar(len(sed_files)) # Loop over SEDs for im, sed_file in enumerate(sed_files): b.update() log.debug('Processing {0}'.format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu') # Convolve for j in range(chunk_size): fluxes[j].central_wavelength = wavelengths[j + jmin] fluxes[j].apertures = apertures fluxes[j].model_names[im] = s.name if n_ap == 1: fluxes[j].flux[im] = s.flux[0, j + jmin] fluxes[j].error[im] = s.error[0, j + jmin] else: fluxes[j].flux[im, :] = s.flux[:, j + jmin] fluxes[j].error[im, :] = s.error[:, j + jmin] for j in range(chunk_size): fluxes[j].sort_to_match(par_table['MODEL_NAME']) fluxes[j].write('{0:s}/convolved/MO{1:03d}.fits'.format( model_dir, j + jmin + 1), overwrite=overwrite) filters['filter'][j + jmin] = "MO{0:03d}".format(j + jmin + 1) return filters
def fit_all_tex(xaxis, cube, cubefrequencies, indices, degeneracies, ecube=None, replace_bad=False): """ Parameters ---------- replace_bad : bool Attempt to replace bad (negative) values with their upper limits? """ tmap = np.empty(cube.shape[1:]) Nmap = np.empty(cube.shape[1:]) yy, xx = np.indices(cube.shape[1:]) pb = ProgressBar(xx.size) count = 0 for ii, jj in (zip(yy.flat, xx.flat)): if any(np.isnan(cube[:, ii, jj])): tmap[ii, jj] = np.nan else: if replace_bad: uplims = nupper_of_kkms( replace_bad, cubefrequencies, einsteinAij[indices], degeneracies, ).value else: uplims = None nuppers = nupper_of_kkms( cube[:, ii, jj], cubefrequencies, einsteinAij[indices], degeneracies, ) if ecube is not None: nupper_error = nupper_of_kkms( ecube[:, ii, jj], cubefrequencies, einsteinAij[indices], degeneracies, ).value uplims = 3 * nupper_error if replace_bad: raise ValueError("replace_bad is ignored now...") else: nupper_error = None fit_result = fit_tex(xaxis, nuppers.value, errors=nupper_error, uplims=uplims) tmap[ii, jj] = fit_result[1].value Nmap[ii, jj] = fit_result[0].value pb.update(count) count += 1 return tmap, Nmap
results = {} if linename in cached_gaussfit_results: print("Loading {0} from cache".format(linename)) results = cached_gaussfit_results[linename] else: print("Fitting {0}, which is not in cache".format(linename)) pb = ProgressBar(cube.shape[0]) for vel,vslice in (zip(cube.spectral_axis, cube)): closest = np.argmin(np.abs(vel-velocities)) if np.abs(velocities[closest] - vel) > vdiff: #print("Skipping velocity {0}, closest is {1} -> {2}".format(vel, closest, # velocities[closest])) pb.update() continue thisvel = velocities[closest].value guess_regs = guesses[thisvel] ampguess = vslice.max().value model_list = [] for reg in guess_regs: p_init = models.Gaussian2D(amplitude=ampguess, x_mean=reg.center.x, y_mean=reg.center.y, x_stddev=bmmaj_px/STDDEV_TO_FWHM*0.75, y_stddev=bmmin_px/STDDEV_TO_FWHM*0.75,
def compute_spatial_distrib(self, radius=None, periodic=True, min_frac=0.8, show_progress=True): ''' Compute the moments over circular region with the specified radius. Parameters ---------- radius : `~astropy.units.Quantity`, optional Override the radius size of the region. periodic : bool, optional Specify whether the boundaries can be wrapped. Default is True. min_frac : float, optional A number between 0 and 1 that sets the minimum fraction of data in each region that are finite. A value of 1.0 requires that no NaNs be in the region. show_progress : bool, optional Show a progress bar during the creation of the covariance matrix. ''' # Require the fraction to be > 0 and <=1 if min_frac <= 0.0 or min_frac > 1.: raise ValueError("min_frac must be larger than 0 and less than" "or equal to 1.") self._mean_array = np.empty(self.data.shape) self._variance_array = np.empty(self.data.shape) self._skewness_array = np.empty(self.data.shape) self._kurtosis_array = np.empty(self.data.shape) # Use the new radius when another given if radius is not None: self.radius = radius # Convert to pixels. We need this to be an integer so round down to # the nearest integer values pix_rad = np.ceil(self._to_pixel(self.radius).value).astype(int) if periodic: pad_img = np.pad(self.data, pix_rad, mode="wrap") pad_weights = np.pad(self.weights, pix_rad, mode="wrap") else: pad_img = np.pad(self.data, pix_rad, padwithnans) pad_weights = np.pad(self.weights, pix_rad, padwithnans) circle_mask = circular_region(pix_rad) if show_progress: bar = ProgressBar((pad_img.shape[0] - 2 * pix_rad) * (pad_img.shape[1] - 2 * pix_rad)) # Loop through every point within the non-padded shape. prod = product(range(pix_rad, pad_img.shape[0] - pix_rad), range(pix_rad, pad_img.shape[1] - pix_rad)) for n, (i, j) in enumerate(prod): img_slice = pad_img[i - pix_rad:i + pix_rad + 1, j - pix_rad:j + pix_rad + 1] wgt_slice = pad_weights[i - pix_rad:i + pix_rad + 1, j - pix_rad:j + pix_rad + 1] valid_img_frac = \ np.isfinite(img_slice).sum() / float(img_slice.size) valid_wgt_frac = \ np.isfinite(wgt_slice).sum() / float(wgt_slice.size) if valid_img_frac < min_frac or valid_wgt_frac < min_frac: self.mean_array[i - pix_rad, j - pix_rad] = np.NaN self.variance_array[i - pix_rad, j - pix_rad] = np.NaN self.skewness_array[i - pix_rad, j - pix_rad] = np.NaN self.kurtosis_array[i - pix_rad, j - pix_rad] = np.NaN else: img_slice = img_slice * circle_mask wgt_slice = wgt_slice * circle_mask moments = compute_moments(img_slice, wgt_slice) self.mean_array[i - pix_rad, j - pix_rad] = moments[0] self.variance_array[i - pix_rad, j - pix_rad] = moments[1] self.skewness_array[i - pix_rad, j - pix_rad] = moments[2] self.kurtosis_array[i - pix_rad, j - pix_rad] = moments[3] if show_progress: bar.update(n + 1)
def _convolve_model_dir_1(model_dir, filters, overwrite=False): for f in filters: if f.name is None: raise Exception("filter name needs to be set") if f.central_wavelength is None: raise Exception("filter central wavelength needs to be set") # Create 'convolved' sub-directory if needed if not os.path.exists(model_dir + '/convolved'): os.mkdir(model_dir + '/convolved') # Find all SED files to convolve sed_files = (glob.glob(model_dir + '/seds/*.fits.gz') + glob.glob(model_dir + '/seds/*/*.fits.gz') + glob.glob(model_dir + '/seds/*.fits') + glob.glob(model_dir + '/seds/*/*.fits')) par_table = load_parameter_table(model_dir) if len(sed_files) == 0: raise Exception("No SEDs found in %s" % model_dir) else: log.info("{0} SEDs found in {1}".format(len(sed_files), model_dir)) # Find out apertures first_sed = SED.read(sed_files[0]) n_ap = first_sed.n_ap apertures = first_sed.apertures # Set up convolved fluxes fluxes = [ ConvolvedFluxes(model_names=np.zeros( len(sed_files), dtype='U30' if six.PY3 else 'S30'), apertures=apertures, initialize_arrays=True) for i in range(len(filters)) ] # Set up list of binned filters binned_filters = [] binned_nu = None # Loop over SEDs b = ProgressBar(len(sed_files)) for im, sed_file in enumerate(sed_files): log.debug('Convolving {0}'.format(os.path.basename(sed_file))) # Read in SED s = SED.read(sed_file, unit_freq=u.Hz, unit_flux=u.mJy, order='nu') # Check if filters need to be re-binned try: assert binned_nu is not None np.testing.assert_array_almost_equal_nulp(s.nu.value, binned_nu.value, 100) except AssertionError: log.info('Rebinning filters') binned_filters = [f.rebin(s.nu) for f in filters] binned_nu = s.nu b.update() # Convolve for i, f in enumerate(binned_filters): fluxes[i].central_wavelength = f.central_wavelength fluxes[i].apertures = apertures fluxes[i].model_names[im] = s.name if n_ap == 1: fluxes[i].flux[im] = np.sum(s.flux * f.response) fluxes[i].error[im] = np.sqrt(np.sum( (s.error * f.response)**2)) else: fluxes[i].flux[im, :] = np.sum(s.flux * f.response, axis=1) fluxes[i].error[im] = np.sqrt( np.sum((s.error * f.response)**2, axis=1)) for i, f in enumerate(binned_filters): fluxes[i].sort_to_match(par_table['MODEL_NAME']) fluxes[i].write(model_dir + '/convolved/' + f.name + '.fits', overwrite=overwrite)
def render_13co(ytcube=yt13co, outdir='yt_renders_13CO', size=512, scale=1200., nframes=180, movie=True, camera_angle=[0, 0, 1], north_vector = [1, 0, 0], rot_vector1=[1,0,0], rot_vector2=[0.5,0.5,0.0], rot_vector3=[0.0,0.5,0.5], ): if not os.path.exists(paths.mpath(outdir)): os.makedirs(paths.mpath(outdir)) tf = yt.ColorTransferFunction([0,30], grey_opacity=True) #tf.map_to_colormap(0.1,5,colormap='Reds') tf.add_gaussian(2, 1, [1.0, 0.8, 0.0, 1.0]) tf.add_gaussian(3, 2, [1.0, 0.5, 0.0, 1.0]) tf.add_gaussian(5, 3, [1.0, 0.0, 0.0, 1.0]) tf.add_gaussian(10, 5, [1.0, 0.0, 0.0, 0.5]) tf.map_to_colormap(10, 30, colormap=red, scale=1) center = ytcube.dataset.domain_dimensions /2. cam = ytcube.dataset.h.camera(center, camera_angle, scale, size, tf, north_vector=north_vector, fields='flux') im = cam.snapshot() #images = [im] if movie: pb = ProgressBar(nframes*2+30) for ii,im in enumerate(cam.rotation(2 * np.pi, nframes/3, rot_vector=rot_vector1)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir,"%04i.png" % (ii))), rescale=False) pb.update(ii) for jj,im in enumerate(cam.rotation(2 * np.pi, nframes/3, rot_vector=rot_vector2)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir,"%04i.png" % (ii+jj))), rescale=False) pb.update(ii+jj) for kk,im in enumerate(cam.rotation(2 * np.pi, nframes/3, rot_vector=rot_vector3)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir,"%04i.png" % (ii+jj+kk))), rescale=False) pb.update(ii+jj+kk) TheBrick = ytcube.world2yt([0.253, 0.016, 35]) for LL, snapshot in enumerate(cam.move_to(TheBrick, 15)): #images.append(snapshot) snapshot.write_png(paths.mpath(os.path.join(outdir,'%04i.png' % (ii+jj+kk+LL))), rescale=False) pb.update(ii+jj+kk+LL) for mm, snapshot in enumerate(cam.zoomin(5, 15)): #images.append(snapshot) snapshot.write_png(paths.mpath(os.path.join(outdir,'%04i.png' % (ii+jj+kk+LL+mm))), rescale=False) pb.update(ii+jj+kk+LL+mm) for nn,im in enumerate(cam.rotation(2*np.pi, nframes/3, rot_vector=rot_vector1)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir,"%04i.png" % (ii+jj+kk+LL+mm+nn))), rescale=False) pb.update(ii+jj+kk+LL+mm+nn) for oo,im in enumerate(cam.rotation(2 * np.pi, nframes/3, rot_vector=rot_vector2)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir,"%04i.png" % (ii+jj+kk+LL+mm+nn+oo))), rescale=False) pb.update(ii+jj+kk+LL+mm+nn+oo) for pp,im in enumerate(cam.rotation(2 * np.pi, nframes/3, rot_vector=rot_vector3)): #images.append(im) im.write_png(paths.mpath(os.path.join(outdir,"%04i.png" % (ii+jj+kk+LL+mm+nn+oo+pp))), rescale=False) pb.update(ii+jj+kk+LL+mm+nn+oo+pp) #save_images(images, paths.mpath(outdir)) pipe = make_movie(paths.mpath(outdir)) return images else: return im
def dataGen2(f): r""" Generates a dictionary of galaxy objects from a file Parameters f - Type: str. The name of the file Returns galaxies - Type: dict. A dictionary of galaxy objects with object IDs as keys """ data = pd.read_csv(f) #print(data['objID']) galaxies = {} if not 'nearbyID' in data.columns: # As of now, this column is always included in datasets not # including object IDs of nearby galaxies and never in # data sets that do include them print("If this shows up, it's a problem") # Debug return ValueError() # Include number of nearby neighbors if it is present if 'modelMag_u' in data.columns: if 'Column1' in data.columns: for i in range(len(data['objID'])): galaxies[int(data['objID'][i])] = galaxy( int(data['objID'][i]), data['ra'][i], data['dec'][i], data['z'][i], 0., 0., data['bpt'][i], u=data['modelMag_u'][i], r=data['modelMag_r'][i], nearby=(int(data['Column1'][i]) - 1)) # Exclude number of nearby neighbors if it is not present else: for i in range(len(data['objID'])): galaxies[int(data['objID'][i])] = galaxy( int(data['objID'][i]), data['ra'][i], data['dec'][i], data['z'][i], 0., 0., data['bpt'][i], u=data['modelMag_u'][i], r=data['modelMag_r'][i]) else: if 'Column1' in data.columns: for i in range(len(data['objID'])): galaxies[int(data['objID'][i])] = galaxy( int(data['objID'][i]), data['ra'][i], data['dec'][i], data['z'][i], 0., 0., data['bpt'][i], nearby=(int(data['Column1'][i]) - 1)) # Exclude number of nearby neighbors if it is not present else: for i in range(len(data['objID'])): galaxies[int(data['objID'][i])] = galaxy( int(data['objID'][i]), data['ra'][i], data['dec'][i], data['z'][i], 0., 0., data['bpt'][i]) # This works slightly different when we want to get IDs of nearby galaxies else: sampleKey = 1237645879551066262 # Debug sampleKey2 = 1237645941824356443 # Debug print("Checkpoint 1") # Debug bar = ProgressBar(len(data['objID'])) for i in range(len(data['objID'])): objid = int(data['objID'][i]) nearID = int(data['nearbyID'][i]) #if data['nearbyID'][i] in data['objID']: if objid in galaxies: print("\nCheckpoint 2") # Debug print("Target ID:", objid) # Debug print("Nearby ID:", nearID) # Debug print("Key 1:", galaxies[objid].nearbyIDs) # Debug galaxies[objid].nearbyIDs.append(nearID) print("Key 1:", galaxies[objid].objId) # Debug print("Key 1:", galaxies[objid].nearbyIDs) # Debug #print("Key 1:", galaxies[sampleKey].nearbyIDs) # Debug if sampleKey2 in galaxies: # Debug print("Found key 2!") # Debug print("Key 2:", galaxies[sampleKey2].nearbyIDs) # Debug else: print("\nCheckpoint 3") # Debug print("Target ID:", objid) # Debug print("Nearby ID:", nearID) # Debug galaxies[objid] = galaxy(objid, data['ra'][i], data['dec'][i], data['z'][i], 0., 0., data['bpt'][i]) #galaxies[objID].nearbyIDs.append(nearID) print("Key 1:", galaxies[sampleKey].nearbyIDs) # Debug if sampleKey2 in galaxies: print("Found key 2!") # Debug print("Key 2:", galaxies[sampleKey2].nearbyIDs) # Debug bar.update() del objid del nearID print("\n") for key in galaxies: # Iterate through dictionary of galaxies galaxies[key].nearby = len( galaxies[key].nearbyIDs ) # Set number of nearby neighbors as length of list of nearby neighbors print("Key 1 before pass:"******"Key 1 Mr before pass:"******"Key 2 before pass:", galaxies[sampleKey2].nearbyIDs) # Debug return galaxies
def render_chem(yth2co321=yth2co321, yth2co303=yth2co303, ytsio=ytsio, outdir='yt_renders_chem3', size=512, scale=1100., nframes=60, north_vector=[1,0,0], rot_vector=[1,0,0], movie=True, camera_angle=[-0.6, 0.4, 0.6]): if not os.path.exists(paths.mpath(outdir)): os.makedirs(paths.mpath(outdir)) tf1 = yt.ColorTransferFunction([0.1,2], grey_opacity=True) #tf1.add_gaussian(0.1,0.05, [1,0,0,1]) #tf1.map_to_colormap(0, 0.5, scale=1, colormap='Reds') tf1.add_gaussian(0.25, 0.01, [1,0,0,1]) tf1.add_step(0.25,0.5,[1,0,0,1]) #tf1.add_step(1,2,[1,1,1,1]) tf1.map_to_colormap(0.5,2,scale=1,colormap=red) tf2 = yt.ColorTransferFunction([0.1,2], grey_opacity=True) #tf2.add_gaussian(0.1,0.05, [0,1,0,1]) #tf2.map_to_colormap(0, 0.5, scale=1, colormap='Greens') tf2.add_gaussian(0.25, 0.01, [0,1,0,1]) tf2.add_step(0.25,0.5,[0,1,0,1]) tf2.map_to_colormap(0.5,2,scale=1,colormap=green) tf3 = yt.ColorTransferFunction([0.1,2], grey_opacity=True) #tf3.add_gaussian(0.1,0.05, [0,0,1,1]) #tf3.map_to_colormap(0, 0.5, scale=1, colormap='Blues') tf3.add_gaussian(0.25, 0.01, [0,0,1,1]) tf3.add_step(0.25,0.5,[0,0,1,1]) tf3.map_to_colormap(0.5,2,scale=1,colormap=blue) center = yth2co303.dataset.domain_dimensions /2. camh2co303 = yth2co303.dataset.h.camera(center, camera_angle, scale, size, tf3, north_vector=north_vector, fields='flux') camh2co321 = yth2co321.dataset.h.camera(center, camera_angle, scale, size, tf2, north_vector=north_vector, fields='flux') camsio = ytsio.dataset.h.camera(center, camera_angle, scale, size, tf1, north_vector=north_vector, fields='flux') imh2co303 = camh2co303.snapshot() imh2co321 = camh2co321.snapshot() imsio = camsio.snapshot() pl.figure(1) pl.clf() pl.imshow(imh2co303+imh2co321+imsio) pl.figure(2) pl.clf() pl.imshow(imh2co303[:,:,:3]+imh2co321[:,:,:3]+imsio[:,:,:3]) if movie: images_h2co303 = [imh2co303] images_h2co321 = [imh2co321] images_sio = [imsio] r1 = camh2co303.rotation(2 * np.pi, nframes, rot_vector=rot_vector) r2 = camh2co321.rotation(2 * np.pi, nframes, rot_vector=rot_vector) r3 = camsio.rotation(2 * np.pi, nframes, rot_vector=rot_vector) pb = ProgressBar(nframes * 3) for (ii,(imh2co303,imh2co321,imsio)) in enumerate(izip(r1, r2, r3)): images_h2co303.append(imh2co303) images_h2co321.append(imh2co321) images_sio.append(imsio) imh2co303=imh2co303.swapaxes(0,1) imh2co303.write_png(paths.mpath(os.path.join(outdir,"h2co303_%04i.png" % (ii))), rescale=False) pb.update(ii*3) imsio=imsio.swapaxes(0,1) imsio.write_png(paths.mpath(os.path.join(outdir,"sio_%04i.png" % (ii))), rescale=False) pb.update(ii*3+1) imh2co321=imh2co321.swapaxes(0,1) imh2co321.write_png(paths.mpath(os.path.join(outdir,"h2co321_%04i.png" % (ii))), rescale=False) pb.update(ii*3+2) pb.next() save_images([i1+i2+i3 for i1,i2,i3 in izip(images_h2co303, images_h2co321, images_sio)], paths.mpath(outdir)) make_movie(paths.mpath(outdir)) return images_h2co303,images_h2co321,images_sio else: return imh2co303,imh2co321,imsio