def test_geomarea_projection(): '''When a ray sees the aperture under an angle the projected aperture size is smaller. This is accounted for by reducing the probability of this photon.''' photons = Table() photons['ra'] = [0., 45., 90., 135., 315.] photons['dec'] = np.zeros(5) photons['origin_coord'] = SkyCoord(photons['ra'], photons['dec'], unit='deg') photons['probability'] = np.ones(5) photons['time'] = np.arange(5) photons['polangle'] = np.zeros(5) fp = FixedPointing(coords=SkyCoord(0., 0., unit='deg')) photons = fp(photons) aper = RectangleAperture() p = aper(photons.copy()) assert np.allclose(p['probability'], [1., 1. / np.sqrt(2), 0, 0, 1. / np.sqrt(2)]) orientation = np.array([[0, 0, 1], [1, 0, 0], [0, 1, 0]]) aper = RectangleAperture(orientation=orientation) p = aper(photons.copy()) assert np.allclose(p['probability'], [0, 1. / np.sqrt(2), 1., 1. / np.sqrt(2), 0.])
def test_pre_post_process(): '''test pre-processing and post-processing in sequences''' def set_energy2(photons): photons['energy'] = 2 return photons def set_energy3(photons): photons['energy'] = 3 return photons def process(photons): photons.meta['mean_en'].append(photons['energy'].mean()) photons = Table({'energy': [1,1,1]}) photons.meta['mean_en'] = [] seq_pre = Sequence(elements=[set_energy2, set_energy3], preprocess_steps=[process]) tpre = seq_pre(photons.copy()) assert np.all(tpre['energy'] == 3) assert np.all(tpre.meta['mean_en'] == [1, 2]) seq_post = Sequence(elements=[set_energy2, set_energy3], postprocess_steps=[process]) tpost = seq_post(photons.copy()) assert np.all(tpost['energy'] == 3) assert np.all(tpost.meta['mean_en'] == [2, 3])
def test_CAT_order_convention(): dirs = np.array([[-1, 0., 0., 0], [-1, 0.01, -0.01, 0], [-1, 0.01, 0.01, 0], [-1, -0.01, 0.01, 0], [-1, -0.01, -0.01, 0]]) photons = Table({ 'pos': np.ones((5, 4)), 'dir': dirs, 'energy': np.ones(5), 'polarization': np.ones(5), 'probability': np.ones(5), }) gp = CATGrating(d=1. / 5000, order_selector=constant_order_factory(5), zoom=2) p5 = gp.process_photons(photons.copy()) gm = CATGrating(d=1. / 5000, order_selector=constant_order_factory(-5), zoom=2) m5 = gm.process_photons(photons.copy()) for g in [gm, gp]: assert np.all( g.order_sign_convention(h2e(photons['dir'])) == np.array( [1, -1, -1, 1, 1])) assert np.all(p5['dir'][1:3, 1] > 0) assert np.all(p5['dir'][3:, 1] < 0) assert np.all(m5['dir'][1:3, 1] < 0) assert np.all(m5['dir'][3:, 1] > 0)
def make_mtl(targets, zcat=None, trim=True): ''' Adds NUMOBS, PRIORITY, and GRAYLAYER columns to a targets table Args: targets : Table with columns TARGETID, DESI_TARGET Optional: zcat : redshift catalog table with columns TARGETID, NUMOBS, Z, ZWARN trim: if True (default), don't include targets that don't need any more observations. If False, include every input target. Returns: MTL Table with targets columns plus * NUMOBS_MORE - number of additional observations requested * PRIORITY - target priority (larger number = higher priority) * GRAYLAYER - can this be observed during gray time? TODO: Check if input targets is ever altered (ist shouldn't...) ''' n = len(targets) targets = Table(targets) if zcat is not None: ztargets = join(targets, zcat, keys='TARGETID', join_type='outer') if ztargets.masked: unobs = ztargets['NUMOBS'].mask ztargets['NUMOBS'][unobs] = 0 else: ztargets = targets.copy() ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32) ztargets['Z'] = -1 * np.ones(n, dtype=np.float32) ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32) ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS']) mtl = targets.copy() mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE'] mtl['PRIORITY'] = calc_priority(ztargets) #- ELGs can be observed during gray time graylayer = np.zeros(n, dtype='i4') iselg = (mtl['DESI_TARGET'] & desi_mask.ELG) != 0 graylayer[iselg] = 1 mtl['GRAYLAYER'] = graylayer if trim: notdone = mtl['NUMOBS_MORE'] > 0 mtl = mtl[notdone] return mtl
def _create_cigale_in(photom_cat: Table, zmin: float = 0.01, zmax: float = 0.35, n_z: int = 35, cigale_input: str = "cigin_minz_zfrb.fits") -> Table: """ Take the photometry table and create a new table with redshifts. For each galaxy, create multiple entries with different redshifts from 0 to 2. These redshifts will be uniformly spaced. Args: photom_cat (Table): Photometry catalog zmin (float, optional): Minimum redshift for analysis. zmax (float, optional): Maximum redshift for analysis. n_z (int, optional): Number of redshift grid points. cigale_input (str, optional): Name of input file to be produced. Returns: stacked_photom (Table): A table with multiple groups, one for each galaxy. Each entry in a group has the same photometry but different redshift values. This way, CIGALE can be run on the same galaxy at multiple redshift guesses in one go. """ # Define z values z_range = np.linspace(zmin, zmax, n_z) photom_cat['redshift'] = z_range[0] # Set up initial redshift value photom_cat['ID'] = photom_cat['ID'].astype(str) # Convert form int to str photom_cat.sort("separation") photom_cat['ID'] = [ ID.zfill(5) + "_{:0.2f}".format(z_range[0]) for ID in photom_cat['ID'] ] # Create new table stacked_photom = photom_cat.copy() for z in z_range[1:]: newphotom = photom_cat.copy() newphotom['redshift'] = z for entry in newphotom: entry['ID'] = entry['ID'].replace("_0.01", "_{:0.2f}".format(z)) stacked_photom = vstack([stacked_photom, newphotom]) # Sort table by ID stacked_photom = stacked_photom.group_by('ID') # Write to disk stacked_photom.write(cigale_input, overwrite=True) print("Wrote to disk {:s}".format(cigale_input)) return stacked_photom
def test_zeros_order(): '''Photons diffracted into order 0 should just pass through''' photons = Table({ 'pos': random((10, 4)) * 10 - 5, 'dir': random((10, 4)) * 10 - 5, 'energy': random(10), 'polarization': random(10), 'probability': np.ones(10), }) # Make sure homogeneous coordiantes are valid # and infall is normal photons['pos'][:, 3] = 1 photons['dir'][:, 1:] = 0 p = photons.copy() def mock_order(x, y, z): return np.zeros_like(x, dtype=np.int), np.ones_like(x) g0 = FlatGrating(d=1. / 500., order_selector=mock_order, zoom=np.array([1., 5., 5.])) p = g0.process_photons(p) # Direction unchanged d_in = h2e(photons['dir']) d_out = h2e(p['dir']) # normalize d_in = d_in / np.sqrt(np.sum(d_in**2, axis=-1))[:, None] d_out = d_out / np.sqrt(np.sum(d_out**2, axis=-1))[:, None] assert np.allclose(d_in, d_out) # all intersection points in y-z plane assert np.allclose(p['pos'][:, 0], 0.) # no offset between old and new ray assert np.allclose( np.cross(h2e(photons['pos']) - h2e(p['pos']), h2e(p['dir'])), 0)
def __init__(self,stripeindex=None): if stripeindex == None: BCfile = MISTFILE_default else: BCfile = '/n/regal/conroy_lab/pac/MISTFILES/MIST_full_{0}.h5'.format(stripeindex) # read in MIST hdf5 table MISTh5 = h5py.File(BCfile,'r') # determine the BC datasets BCTableList = [x for x in MISTh5.keys() if x[:3] == 'BC_'] # read in each BC dataset and pull the photometric information for BCT in BCTableList: BCTABLE = Table(np.array(MISTh5[BCT])) if BCT == BCTableList[0]: BC = BCTABLE.copy() else: BCTABLE.remove_columns(['Teff', 'logg', '[Fe/H]', 'Av', 'Rv']) BC = hstack([BC,BCTABLE]) BC_AV0 = BC[BC['Av'] == 0.0] self.bands = BC.keys() [self.bands.remove(x) for x in ['Teff', 'logg', '[Fe/H]', 'Av', 'Rv']] self.redintr = LinearNDInterpolator( (BC['Teff'],BC['logg'],BC['[Fe/H]'],BC['Av']), np.stack([BC[bb] for bb in self.bands],axis=1), rescale=True ) self.redintr_0 = LinearNDInterpolator( (BC_AV0['Teff'],BC_AV0['logg'],BC_AV0['[Fe/H]']), np.stack([BC_AV0[bb] for bb in self.bands],axis=1), rescale=True )
def test_zeros_order(): '''Photons diffracted into order 0 should just pass through''' photons = Table({'pos': random((10, 4)) * 10 - 5, 'dir': random((10, 4)) * 10 - 5, 'energy': random(10), 'polarization': random(10), 'probability': np.ones(10), }) # Make sure homogeneous coordiantes are valid # and infall is normal photons['pos'][:, 3] = 1 photons['dir'][:, 1:] = 0 p = photons.copy() def mock_order(x, y, z): return np.zeros_like(x, dtype=np.int), np.ones_like(x) g0 = FlatGrating(d=1./500., order_selector=mock_order, zoom=np.array([1., 5., 5.])) p = g0.process_photons(p) # Direction unchanged d_in = h2e(photons['dir']) d_out = h2e(p['dir']) # normalize d_in = d_in / np.sqrt(np.sum(d_in**2, axis=-1))[:, None] d_out = d_out / np.sqrt(np.sum(d_out**2, axis=-1))[:, None] assert np.allclose(d_in, d_out) # all intersection points in y-z plane assert np.allclose(p['pos'][:, 0], 0.) # no offset between old and new ray assert np.allclose(np.cross(h2e(photons['pos']) - h2e(p['pos']), h2e(p['dir'])), 0)
def main(): path = "." pattern = "" if len(sys.argv) == 3: path = sys.argv[1] pattern = sys.argv[2] else: pattern = sys.argv[1] for file in os.listdir(path): if fnmatch.fnmatch(file, pattern): data = Table(Table.read(os.path.join(path, file), format="ascii"), masked=True) orig_mask = (data['ivar'] == 0) data.mask = [(data['ivar'] == 0)]*len(data.columns) idstr = file[:file.rfind('.')] test_data = data.copy() test_data['flux'] = minimize(test_data['wavelength'], test_data['flux'], [100, 200, 300], 0, start_ind=split_noisy_app) filtered = filter_and_subtract(test_data['flux'], test_data['wavelength'], 201, 24) test_data['flux'] = np.ma.min(np.ma.vstack([test_data['flux'], filtered]), axis=0) filtered = filter_and_subtract(test_data['flux'], test_data['wavelength'], 161, 18) test_data['flux'] = np.ma.min(np.ma.vstack([test_data['flux'], filtered]), axis=0) continuum = split_spectrum(test_data['wavelength'], test_data['flux']) wo_continuum = data['flux'] - continuum save_data(data['wavelength'], wo_continuum, continuum, data['ivar'], orig_mask, idstr)
def remove_duplicates(tab: Table, idcol: str) -> Table: """ In an astropy table if there are duplicate entries, remove the duplicates. Generally, these will be duplicate objects (i.e. multiple observations of same object ID or the same entry repeated multiple times from cross-matching.) Args: tab (Table): A table of entries. idcol (str): A column name that has unique ids for each table entry. Returns: unique_tab (Table): A table with only the unique ids. """ unique_tab = tab.copy() assert isinstance(unique_tab, Table), "Please provide an astropy table." assert isinstance(idcol, str), "Please provide a valid column name." assert idcol in tab.colnames, "{} not a column in the given table".format( idcol) # Sort entries first. unique_tab.sort(idcol) # Get the duplicates. duplicate_ids = np.where( unique_tab[1:][idcol] == unique_tab[:-1][idcol])[0] + 1 unique_tab.remove_rows(duplicate_ids) return unique_tab
def test_zeros_order(): '''Photons diffracted into order 0 should just pass through''' dir = random((10, 4)) dir[0, :] = dir[0, :] * 10 # make sure mostion is mostly along x so they hit grating photons = Table({'pos': random((10, 4)) - 1, 'dir': dir, 'energy': random(10), 'polarization': polarization_vectors(dir, np.random.rand(10)), 'probability': np.ones(10), }) # Make sure homogeneous coordiantes are valid # and infall is normal photons['pos'][:, 3] = 1 photons['dir'][:, 1:] = 0 p = photons.copy() g0 = FlatGrating(d=1./500., order_selector=OrderSelector([0]), zoom=np.array([1., 50., 50.])) p = g0(p) # Direction unchanged d_in = h2e(photons['dir']) d_out = h2e(p['dir']) # normalize d_in = d_in / np.sqrt(np.sum(d_in**2, axis=-1))[:, None] d_out = d_out / np.sqrt(np.sum(d_out**2, axis=-1))[:, None] assert np.allclose(d_in, d_out) # all intersection points in y-z plane assert np.allclose(p['pos'][:, 0], 0.) # no offset between old and new ray assert np.allclose(np.cross(h2e(photons['pos']) - h2e(p['pos']), h2e(p['dir'])), 0)
def test_psf_photometry_oneiter(sigma_psf, sources): """ Tests in an image with a group of two overlapped stars and an isolated one. """ img_shape = (32, 32) # generate image with read-out noise (Gaussian) and # background noise (Poisson) image = (make_gaussian_sources(img_shape, sources) + make_noise_image(img_shape, type='poisson', mean=6., random_state=1) + make_noise_image(img_shape, type='gaussian', mean=0., stddev=2., random_state=1)) cp_image = image.copy() sigma_clip = SigmaClip(sigma=3.) bkgrms = StdBackgroundRMS(sigma_clip) std = bkgrms(image) phot_objs = make_psf_photometry_objs(std, sigma_psf) for phot_proc in phot_objs: result_tab = phot_proc(image) residual_image = phot_proc.get_residual_image() assert_allclose(result_tab['x_fit'], sources['x_mean'], rtol=1e-1) assert_allclose(result_tab['y_fit'], sources['y_mean'], rtol=1e-1) assert_allclose(result_tab['flux_fit'], sources['flux'], rtol=1e-1) assert_array_equal(result_tab['id'], sources['id']) assert_array_equal(result_tab['group_id'], sources['group_id']) assert_allclose(np.mean(residual_image), 0.0, atol=1e1) # test fixed photometry phot_proc.psf_model.x_0.fixed = True phot_proc.psf_model.y_0.fixed = True pos = Table(names=['x_0', 'y_0'], data=[sources['x_mean'], sources['y_mean']]) cp_pos = pos.copy() result_tab = phot_proc(image, pos) residual_image = phot_proc.get_residual_image() assert_array_equal(result_tab['x_fit'], sources['x_mean']) assert_array_equal(result_tab['y_fit'], sources['y_mean']) assert_allclose(result_tab['flux_fit'], sources['flux'], rtol=1e-1) assert_array_equal(result_tab['id'], sources['id']) assert_array_equal(result_tab['group_id'], sources['group_id']) assert_allclose(np.mean(residual_image), 0.0, atol=1e1) # make sure image is note overwritten assert_array_equal(cp_image, image) # make sure initial guess table is not modified assert_array_equal(cp_pos, pos) # resets fixed positions phot_proc.psf_model.x_0.fixed = False phot_proc.psf_model.y_0.fixed = False
def _likelihood_ratio(self, pidx, sidx, d2d): """ Estimates the likelihood ratio for all counterparts and for each magnitude band. Parameters ---------- pidx : numpy ``ndarray`` Indexes of the primary sources with counterparts in the secondary catalogue. sidx : numpy ``ndarray`` Indexes of the counterparts in the secondary catalogue. d2d : numpy ``ndarray`` Distance between the primary source and the counterpart in the secondary catalogue. Return ------ lr : Astropy ``Table`` Table with the likelihood ratios for each counterpart and each available magnitude in the secondary catalogue. """ pcat_idcol = 'SRCID_{}'.format(self.pcat.name) scat_idcol = 'SRCID_{}'.format(self.scat.name) drcol = 'Separation_{}_{}'.format(self.pcat.name, self.scat.name) lr = Table() lr[pcat_idcol] = self.pcat.ids[pidx] lr[scat_idcol] = self.scat.ids[sidx] lr[drcol] = d2d.to(u.arcsec) #print(self.pcat.ids[pidx]) #print(self.scat.ids[sidx]) #print(d2d.to(u.arcsec)) # AGE: if cata has no ids it fails with lr['ncat']=2 lr['ncat'] = [2] * len( lr[pcat_idcol] ) lr['PEF'] = self._pos_err_function(d2d, pidx, sidx) #self._qnterms(lr, self.scat.mags[sidx]) self._lrND(lr, self.scat.mags[sidx]) # For estimating the reliability, the table has to be grouped. # Note that this changes the order of the rows!!! lr = lr.group_by(pcat_idcol) self._reliabilityND(lr) self._p_any_bestND(lr) lr_all = lr.copy() final_columns = [pcat_idcol, scat_idcol, drcol, 'ncat', 'LR_BEST', 'REL_BEST', 'LR_BEST_MAG', 'prob_has_match', 'prob_this_match', ] lr.keep_columns(final_columns) return lr, lr_all
def stats_rndmatch(self, match, ntest=1, ncutoff=101, mincutoff=0.0, maxcutoff=10.0, plot_to_file=None, **kwargs): """ Calculates match statistics (completness and reliability), using a random match, for a range of LR thresholds. This can be used later to select the optimal threshold. """ fstats = None for _ in range(ntest): match_rnd = self._match_rndcat(**kwargs) # TODO: LR ouput should be improved to simplify these selection masks mask = np.logical_and(match['ncat'] == 2, match['match_flag'] == 1) p_any0 = match[self._cutoff_column][mask] # Add sources with no matches size = len(np.where(match['ncat'] == 1)[0]) - len(p_any0) p_any0 = np.concatenate((np.array(p_any0), np.zeros(size))) mask = np.logical_and(match_rnd['ncat'] == 2, match_rnd['match_flag'] == 1) p_any0_offset = match_rnd[self._cutoff_column][mask] # Add sources with no matches size = len( np.where(match_rnd['ncat'] == 1)[0]) - len(p_any0_offset) p_any0_offset = np.concatenate( (np.array(p_any0_offset), np.zeros(size))) cutoffs = np.linspace(mincutoff, maxcutoff, num=ncutoff) stats = Table() stats['cutoff'] = cutoffs stats['completeness'] = [(p_any0 > c).mean() for c in cutoffs] stats['error_rate'] = [(p_any0_offset > c).mean() for c in cutoffs] stats['reliability'] = 1 - stats['error_rate'] stats['CR'] = stats['completeness'] + stats['reliability'] if fstats is None: fstats = stats.copy() else: for col in fstats.colnames[1:]: fstats[col] += stats[col] for col in fstats.colnames[1:]: fstats[col] = fstats[col] / ntest if plot_to_file is not None: self._plot_stats(fstats, plot_to_file) return fstats
def run_photometry( image: np.ndarray, input_table: Table, filename: str = '?', config=Config.instance()) -> Union[PhotometryResult, str]: """ apply EPSF fitting photometry to a testimage :param input_table: :param image: :param filename: :param config: instance of Config containing all processing parameters :return: PhotometryResult, (image, input_table, result_table, epsf, star_guesses) """ print(f'starting job on image {filename} with {config}') finder = get_finder(image, config) # TODO should this also be done using the catalogue positions? # TODO can we somehow sort the stars according to usefulness? star_guesses = make_stars_guess( image, finder, cutout_size=config.cutout_size)[:config.max_epsf_stars] if len(star_guesses) < config.max_epsf_stars: print('Warning: found less stars than config.stars_to_keep') epsf = make_epsf_fit(star_guesses, iters=config.epsfbuilder_iters, oversampling=config.oversampling, smoothing_kernel=config.smoothing, epsf_guess=config.epsf_guess) if config.use_catalogue_positions: guess_table = input_table.copy() guess_table = cut_edges(guess_table, config.cutout_size, image.shape[0]) guess_table.rename_columns(['x', 'y'], ['x_0', 'y_0']) guess_table['x_0'] += np.random.uniform(-0.2, +0.2, size=len(guess_table['x_0'])) guess_table['y_0'] += np.random.uniform(-0.2, +0.2, size=len(guess_table['y_0'])) else: guess_table = None result_table = do_photometry_epsf(image, epsf, finder, initial_guess=guess_table, config=config) result_table['m'] = flux_to_magnitude(result_table['flux_fit']) return PhotometryResult(image, input_table, result_table, epsf, star_guesses, config, filename)
def test_psf_photometry_oneiter(sigma_psf, sources): """ Tests in an image with a group of two overlapped stars and an isolated one. """ img_shape = (32, 32) # generate image with read-out noise (Gaussian) and # background noise (Poisson) image = (make_gaussian_sources(img_shape, sources) + make_noise_image( img_shape, type='poisson', mean=6., random_state=1) + make_noise_image( img_shape, type='gaussian', mean=0., stddev=2., random_state=1)) cp_image = image.copy() sigma_clip = SigmaClip(sigma=3.) bkgrms = StdBackgroundRMS(sigma_clip) std = bkgrms(image) phot_objs = make_psf_photometry_objs(std, sigma_psf) for phot_proc in phot_objs: result_tab = phot_proc(image) residual_image = phot_proc.get_residual_image() assert_allclose(result_tab['x_fit'], sources['x_mean'], rtol=1e-1) assert_allclose(result_tab['y_fit'], sources['y_mean'], rtol=1e-1) assert_allclose(result_tab['flux_fit'], sources['flux'], rtol=1e-1) assert_array_equal(result_tab['id'], sources['id']) assert_array_equal(result_tab['group_id'], sources['group_id']) assert_allclose(np.mean(residual_image), 0.0, atol=1e1) # test fixed photometry phot_proc.psf_model.x_0.fixed = True phot_proc.psf_model.y_0.fixed = True pos = Table(names=['x_0', 'y_0'], data=[sources['x_mean'], sources['y_mean']]) cp_pos = pos.copy() result_tab = phot_proc(image, pos) residual_image = phot_proc.get_residual_image() assert_array_equal(result_tab['x_fit'], sources['x_mean']) assert_array_equal(result_tab['y_fit'], sources['y_mean']) assert_allclose(result_tab['flux_fit'], sources['flux'], rtol=1e-1) assert_array_equal(result_tab['id'], sources['id']) assert_array_equal(result_tab['group_id'], sources['group_id']) assert_allclose(np.mean(residual_image), 0.0, atol=1e1) # make sure image is not overwritten assert_array_equal(cp_image, image) # make sure initial guess table is not modified assert_array_equal(cp_pos, pos) # resets fixed positions phot_proc.psf_model.x_0.fixed = False phot_proc.psf_model.y_0.fixed = False
def test_order_convention(): dirs = np.zeros((3, 4)) dirs[1, 2] = 0.01 dirs[2, 2] = -0.01 dirs[:, 0] = - 1 photons = Table({'pos': np.ones((3, 4)), 'dir': dirs, 'energy': np.ones(3), 'polarization': polarization_vectors(dirs, np.ones(3)), 'probability': np.ones(3), }) gp = FlatGrating(d=1./500, order_selector=OrderSelector([1]), zoom=2) p1 = gp(photons.copy()) gm = FlatGrating(d=1./500, order_selector=OrderSelector([-1]), zoom=2) m1 = gm(photons.copy()) assert np.all(p1['order'] == 1) assert np.all(m1['order'] == -1) # intersection point with grating cannot depend on order assert np.all(p1['pos'].data == m1['pos'].data)
def test_order_convention(): dirs = np.zeros((3, 4)) dirs[1, 2] = 0.01 dirs[2, 2] = -0.01 dirs[:, 0] = -1 photons = Table({ 'pos': np.ones((3, 4)), 'dir': dirs, 'energy': np.ones(3), 'polarization': polarization_vectors(dirs, np.ones(3)), 'probability': np.ones(3), }) gp = FlatGrating(d=1. / 500, order_selector=OrderSelector([1]), zoom=2) p1 = gp(photons.copy()) gm = FlatGrating(d=1. / 500, order_selector=OrderSelector([-1]), zoom=2) m1 = gm(photons.copy()) assert np.all(p1['order'] == 1) assert np.all(m1['order'] == -1) # intersection point with grating cannot depend on order assert np.all(p1['pos'].data == m1['pos'].data)
def test_CAT_order_convention(): dirs = np.zeros((3, 4)) dirs[:, 0] = -1. dirs[1, 2]= 0.01 dirs[2, 2]= -0.01 photons = Table({'pos': np.ones((3, 4)), 'dir': dirs, 'energy': np.ones(3), 'polarization': np.ones(3), 'probability': np.ones(3), }) gp = CATGrating(d=1./5000, order_selector=constant_order_factory(5), zoom=2) p5 = gp.process_photons(photons.copy()) gm = CATGrating(d=1./5000, order_selector=constant_order_factory(-5), zoom=2) m5 = gm.process_photons(photons.copy()) for g in [gm, gp]: assert np.all(g.order_sign_convention(h2e(photons['dir'])) == np.array([1, 1, -1])) assert p5['dir'][1, 1] > 0 assert p5['dir'][2, 1] < 0 assert m5['dir'][1, 1] < 0 assert m5['dir'][2, 1] > 0
def test_CAT_order_convention(): dirs = np.array([[-1, 0., 0., 0], [-1, 0.01, -0.01, 0], [-1, 0.01, 0.01, 0], [-1, -0.01, 0.01, 0], [-1, -0.01, -0.01, 0]]) photons = Table({'pos': np.ones((5, 4)), 'dir': dirs, 'energy': np.ones(5), 'polarization': np.ones(5), 'probability': np.ones(5), }) gp = CATGrating(d=1./5000, order_selector=constant_order_factory(5), zoom=2) p5 = gp.process_photons(photons.copy()) gm = CATGrating(d=1./5000, order_selector=constant_order_factory(-5), zoom=2) m5 = gm.process_photons(photons.copy()) for g in [gm, gp]: assert np.all(g.order_sign_convention(h2e(photons['dir'])) == np.array([1, -1, -1, 1, 1])) assert np.all(p5['dir'][1:3, 1] > 0) assert np.all(p5['dir'][3:, 1] < 0) assert np.all(m5['dir'][1:3, 1] < 0) assert np.all(m5['dir'][3:, 1] > 0)
def test_fit_drop_empty(mock_fits_wcs): t0 = Table([[], []], names=('x', 'y')) t1 = Table([[1], [3]], names=('x', 'y')) wcscats = [ FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t0.copy(), 'group_id': 1} ), FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t1.copy(), 'group_id': 2} ), FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t0.copy(), 'group_id': 2} ), FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t0.copy(), 'group_id': 3} ), FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t0.copy(), 'group_id': 3} ), FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t1.copy(), 'group_id': 4} ), FITSWCS( copy.deepcopy(mock_fits_wcs), meta={'catalog': t1.copy(), 'group_id': 4} ) ] align_wcs(wcscats, fitgeom='shift') status = [w.meta.get('fit_info')['status'] for w in wcscats] assert status[0] == 'FAILED: empty source catalog' assert status[3] == 'FAILED: empty source catalog' assert status[4] == 'FAILED: empty source catalog' if status[1] == 'SUCCESS': assert status[2] == 'SUCCESS' assert status[5] == 'REFERENCE' assert status[6] == 'REFERENCE' elif status[1] == 'REFERENCE': assert status[2] == 'REFERENCE' assert status[5] == 'SUCCESS' assert status[6] == 'SUCCESS' else: assert False
def test_geomarea_projection(): '''When a ray sees the aperture under an angle the projected aperture size is smaller. This is accounted for by reducing the probability of this photon.''' photons = Table() photons['ra'] = [0., 45., 90., 135., 315.] photons['dec'] = np.zeros(5) photons['origin_coord'] = SkyCoord(photons['ra'], photons['dec'], unit='deg') photons['probability'] = np.ones(5) photons['time'] = np.arange(5) photons['polangle'] = np.zeros(5) fp = FixedPointing(coords=SkyCoord(0., 0., unit='deg')) photons = fp(photons) aper = RectangleAperture() p = aper(photons.copy()) assert np.allclose(p['probability'], [1., 1./np.sqrt(2), 0, 0, 1./np.sqrt(2)]) orientation = np.array([[0, 0, 1],[1, 0, 0],[0, 1, 0]]) aper = RectangleAperture(orientation=orientation) p = aper(photons.copy()) assert np.allclose(p['probability'], [ 0, 1./np.sqrt(2), 1., 1./np.sqrt(2), 0.])
def test_CAT_order_convention(): dirs = np.array([[-1, 0., 0., 0], [-1, 0.01, -0.01, 0], [-1, 0.01, 0.01, 0], [-1, -0.01, 0.01, 0], [-1, -0.01, -0.01, 0]]) photons = Table({ 'pos': np.ones((5, 4)), 'dir': dirs, 'energy': np.ones(5), 'polarization': polarization_vectors(dirs, np.arange(5)), 'probability': np.ones(5), }) gp = CATGrating(d=1. / 5000, order_selector=OrderSelector([5]), zoom=2) p5 = gp(photons.copy()) gm = CATGrating(d=1. / 5000, order_selector=OrderSelector([-5]), zoom=2) m5 = gm(photons.copy()) for g in [gm, gp]: e_groove, e_perp, n = g.e_groove_coos(np.zeros((5, 2))) signs = g.order_sign_convention(photons['dir'], e_perp) assert np.all(signs == np.array([1, 1, 1, -1, -1])) assert np.all(p5['dir'][1:3, 1] > 0) assert np.all(p5['dir'][3:, 1] < 0) assert np.all(m5['dir'][1:3, 1] < 0) assert np.all(m5['dir'][3:, 1] > 0)
def test_CAT_order_convention(): dirs = np.array([[-1, 0., 0., 0], [-1, 0.01, -0.01, 0], [-1, 0.01, 0.01, 0], [-1, -0.01, 0.01, 0], [-1, -0.01, -0.01, 0]]) photons = Table({'pos': np.ones((5, 4)), 'dir': dirs, 'energy': np.ones(5), 'polarization': polarization_vectors(dirs, np.arange(5)), 'probability': np.ones(5), }) gp = CATGrating(d=1./5000, order_selector=OrderSelector([5]), zoom=2) p5 = gp(photons.copy()) gm = CATGrating(d=1./5000, order_selector=OrderSelector([-5]), zoom=2) m5 = gm(photons.copy()) for g in [gm, gp]: e_groove, e_perp, n = g.e_groove_coos(np.zeros((5, 2))) signs = g.order_sign_convention(photons['dir'], e_perp) assert np.all(signs == np.array([1, 1, 1, -1, -1])) assert np.all(p5['dir'][1:3, 1] > 0) assert np.all(p5['dir'][3:, 1] < 0) assert np.all(m5['dir'][1:3, 1] < 0) assert np.all(m5['dir'][3:, 1] > 0)
def generate_evolution_tables(self): for pairnum, filnums in self.pairings.items(): if pairnum == 0: continue current_calibs = Table( self.coarse_calibration_coefs[pairnum].copy()) past_calibs = Table(self.coarse_calibration_coefs[pairnum - 1].copy()) for column in current_calibs.colnames: current_calibs[ column] = current_calibs[column] - past_calibs[column] self.evolution_in_coarse_coefs[pairnum] = current_calibs.copy()
def stats_rndmatch(self, match, ntest=1, ncutoff=101, mincutoff=0.0, maxcutoff=1.0, plot_to_file=None, **kwargs): """ Calculates match statistics (completness and reliability), using a random match, for a range of thresholds. This can be used later to select the optimal threshold. """ # TODO: estimate uncertainties in the final statistics using e.g. bootstraping fstats = None for _ in range(ntest): match_rnd = self._match_rndcat(**kwargs) mask = match['match_flag'] == 1 p_any0 = match[self._cutoff_column][mask] mask = match_rnd['match_flag'] == 1 p_any0_offset = match_rnd[self._cutoff_column][mask] cutoffs = np.linspace(mincutoff, maxcutoff, num=ncutoff) stats = Table() stats['cutoff'] = cutoffs stats['completeness'] = [(p_any0 > c).mean() for c in cutoffs] stats['error_rate'] = [(p_any0_offset > c).mean() for c in cutoffs] stats['reliability'] = 1 - stats['error_rate'] stats['CR'] = stats['completeness'] + stats['reliability'] if fstats is None: fstats = stats.copy() else: for col in fstats.colnames[1:]: fstats[col] += stats[col] for col in fstats.colnames[1:]: fstats[col] = fstats[col] / ntest if plot_to_file is not None: self._plot_stats(fstats, plot_to_file) return fstats
def generate_times_line_emission(spectra: Table, spectra_times: Table, verbose: bool = False): """ Given a finished timeseries of spectra, produce the total line emission at each observation Arguments: spectra (Table): The time-series of spectra spectra_times (Table): The times at which the spectra are taken (i.e. the fake observations) verbose (bool): Whether or not to report on the total line variation Return: Table: A table with the line emission for each timestep """ line_times = spectra_times.copy(copy_data=True) line_times['time'] = line_times['time'].quantity.to(u.s) line_times['line'] = np.zeros(len(line_times)) # Integrated flux error dL = SQRT(dBin1^2 + dBin2^2 + ...) # dL = SQRT(N_Bins * dBin^2) = SQRT(N_Bins) * dBin, we divide by SQRT(N_Bins) line_times['line_error'] = np.zeros(len(line_times)) line_times['line_error'] = np.sqrt(len(spectra)) * spectra['error'][0] # For each spectrum in the output, sum the total emission # This obviously only works for line spectra! for step in range(0, len(line_times)): line_times['line'][step] = np.sum(spectra[spectra.colnames[5 + step]]) if verbose: print("Variation is: {}".format( np.amax(line_times['line']) - np.amin(line_times['line']))) # # spec_max = np.amax(line_times['line']) # spec_min = np.amin(line_times['line']) # error = np.array(line_times['line_error']) / (spec_max - spec_min) # error = (error * 9) # value = (np.array(line_times['line']) - spec_min) / (spec_max - spec_min) # value = (value * 9) + 1 # # line_times['line_error'] = error # line_times['line'] = value return line_times
def test_jitter(): '''test size and randomness of jitter''' n = 100000 ra = np.random.rand(n) * 2 * np.pi # Note that my rays are not evenly distributed on the sky. # No need to be extra fancy here, it's probably even better for # a test to have more scrutiny around the poles. dec = (np.random.rand(n) * 2. - 1.) / 2. * np.pi time = np.arange(n) pol = np.ones_like(ra) prob = np.ones_like(ra) photons = Table([ra, dec, time, pol, prob], names=['ra', 'dec', 'time', 'polangle', 'probability']) fixed = FixedPointing(coords=SkyCoord(25., -10., unit='deg')) jittered = JitterPointing(coords=SkyCoord(25., -10., unit='deg'), jitter=1. * u.arcsec) p_fixed = fixed(photons.copy()) p_jitter = jittered(photons) assert np.allclose(np.linalg.norm(p_fixed['dir'], axis=1), 1.) assert np.allclose(np.linalg.norm(p_jitter['dir'], axis=1), 1.) prod = np.sum(p_fixed['dir'] * p_jitter['dir'], axis=1) # sum can give values > 1 due to rounding errors # That would make arccos fail, so catch those here ind = prod > 1. if ind.sum() > 0: prod[ind] = 1. alpha = np.arccos(prod) # in this formula alpha will always be the abs(angle). # Flip some signs to recover input normal distribtution. alpha *= np.sign(np.random.rand(n) - 0.5) # center? assert np.abs(np.mean(alpha)) * 3600. < 0.01 # Is right size? assert np.std(np.rad2deg(alpha)) > (0.9 / 3600.) assert np.std(np.rad2deg(alpha)) < (1.1 / 3600.) # Does it affect y and z independently? coeff, p = pearsonr(p_fixed['dir'][:, 1] - p_jitter['dir'][:, 1], p_fixed['dir'][:, 2] - p_jitter['dir'][:, 2]) assert abs(coeff) < 0.01
def test_jitter(): '''test size and randomness of jitter''' n = 100000 ra = np.random.rand(n) * 2 * np.pi # Note that my rays are not evenly distributed on the sky. # No need to be extra fancy here, it's probably even better for # a test to have more scrutiny around the poles. dec = (np.random.rand(n) * 2. - 1.) / 2. * np.pi time = np.arange(n) pol = np.ones_like(ra) prob = np.ones_like(ra) photons = Table([ra, dec, time, pol, prob], names=['ra', 'dec', 'time', 'polangle', 'probability']) fixed = FixedPointing(coords = SkyCoord(25., -10., unit='deg')) jittered = JitterPointing(coords = SkyCoord(25., -10., unit='deg'), jitter=1. * u.arcsec) p_fixed = fixed(photons.copy()) p_jitter = jittered(photons) assert np.allclose(np.linalg.norm(p_fixed['dir'], axis=1), 1.) assert np.allclose(np.linalg.norm(p_jitter['dir'], axis=1), 1.) prod = np.sum(p_fixed['dir'] * p_jitter['dir'], axis=1) # sum can give values > 1 due to rounding errors # That would make arccos fail, so catch those here ind = prod > 1. if ind.sum() > 0: prod[ind] = 1. alpha = np.arccos(prod) # in this formula alpha will always be the abs(angle). # Flip some signs to recover input normal distribtution. alpha *= np.sign(np.random.rand(n)-0.5) # center? assert np.abs(np.mean(alpha)) * 3600. < 0.01 # Is right size? assert np.std(np.rad2deg(alpha)) > (0.9 / 3600.) assert np.std(np.rad2deg(alpha)) < (1.1 / 3600.) # Does it affect y and z independently? coeff, p = pearsonr(p_fixed['dir'][:, 1] - p_jitter['dir'][:, 1], p_fixed['dir'][:, 2] - p_jitter['dir'][:, 2]) assert abs(coeff) < 0.01
def apply_spectra_error(spectra: Table): """ Given a timeseries of spectra with an 'error' column, creates a copy and applies random normally-distributed errors to the values at each timestep. Arguments: spectra (Table): Returns: Table: A copy of the input spectra with the errors applied """ # Now we have the final spectra, create clean copies then add the experimental errors clean_copy = spectra.copy(copy_data=True) for column in spectra.colnames[5:]: # For each spectrum if 'value' in column: for j in range(0, len(spectra)): # For each wavelength bin in each spectrum, add a random error spectra[column][j] += normal(scale=spectra['error'][j]) return clean_copy
def match_observation_to_source(reference_catalog: Table, photometry_result: Table) \ -> Table: # TODO is this really necessary? there seems to be x_0, y_0 in results, # can use this in cases where the guess is known """ Match the closest points in a photometry catalogue to the input catalogue :param reference_catalog: Table containing input positions in 'x' and 'y' columns :param photometry_result: Table containing measured positions in 'x_fit' and 'y_fit' columns :return: photometry_result updated with 'x_orig', 'y_orig' and 'offset' (euclidean distance) columns """ from scipy.spatial import cKDTree x_y_pixel = np.array((reference_catalog['x'], reference_catalog['y'], reference_catalog['m'])).T # noinspection PyArgumentList lookup_tree = cKDTree(x_y_pixel[:, :2]) # only feed x and y to the lookup tree photometry_result = photometry_result.copy() photometry_result['x_orig'] = np.nan photometry_result['y_orig'] = np.nan photometry_result['m_orig'] = np.nan photometry_result['offset'] = np.nan seen_indices = set() for row in photometry_result: # noinspection PyUnresolvedReferences dist, index = lookup_tree.query((row['x_fit'], row['y_fit'])) # if index in seen_indices: # print('Warning: multiple match for source') # TODO make this message more useful/use warning module seen_indices.add(index) row['x_orig'] = x_y_pixel[index, 0] row['y_orig'] = x_y_pixel[index, 1] row['m_orig'] = x_y_pixel[index, 2] photometry_result['x_offset'] = photometry_result['x_fit'] - photometry_result['x_orig'] photometry_result['y_offset'] = photometry_result['y_fit'] - photometry_result['y_orig'] photometry_result['offset'] = np.sqrt(photometry_result['x_offset']**2 + photometry_result['y_offset']**2) return photometry_result
def test_zeros_order(): '''Photons diffracted into order 0 should just pass through''' dir = random((10, 4)) dir[0, :] = dir[ 0, :] * 10 # make sure mostion is mostly along x so they hit grating photons = Table({ 'pos': random((10, 4)) - 1, 'dir': dir, 'energy': random(10), 'polarization': polarization_vectors(dir, np.random.rand(10)), 'probability': np.ones(10), }) # Make sure homogeneous coordiantes are valid # and infall is normal photons['pos'][:, 3] = 1 photons['dir'][:, 1:] = 0 p = photons.copy() g0 = FlatGrating(d=1. / 500., order_selector=OrderSelector([0]), zoom=np.array([1., 50., 50.])) p = g0(p) # Direction unchanged d_in = h2e(photons['dir']) d_out = h2e(p['dir']) # normalize d_in = d_in / np.sqrt(np.sum(d_in**2, axis=-1))[:, None] d_out = d_out / np.sqrt(np.sum(d_out**2, axis=-1))[:, None] assert np.allclose(d_in, d_out) # all intersection points in y-z plane assert np.allclose(p['pos'][:, 0], 0.) # no offset between old and new ray assert np.allclose( np.cross(h2e(photons['pos']) - h2e(p['pos']), h2e(p['dir'])), 0)
def make_mtl(targets, zcat=None, trim=False): """Adds NUMOBS, PRIORITY, and GRAYLAYER columns to a targets table. Parameters ---------- targets : :class:`~astropy.table.Table` A table with columns ``TARGETID``, ``DESI_TARGET``. zcat : :class:`~astropy.table.Table`, optional Redshift catalog table with columns ``TARGETID``, ``NUMOBS``, ``Z``, ``ZWARN``. trim : :class:`bool`, optional If ``True``, don't include targets that don't need any more observations. If ``False`` (default), include every input target. Returns ------- :class:`~astropy.table.Table` MTL Table with targets columns plus * NUMOBS_MORE - number of additional observations requested * PRIORITY - target priority (larger number = higher priority) * GRAYLAYER - can this be observed during gray time? Notes ----- TODO: Check if input targets is ever altered (it shouldn't...). """ n = len(targets) targets = Table(targets) if zcat is not None: ztargets = join(targets, zcat['TARGETID', 'NUMOBS', 'Z', 'ZWARN'], keys='TARGETID', join_type='outer') if ztargets.masked: unobs = ztargets['NUMOBS'].mask ztargets['NUMOBS'][unobs] = 0 else: ztargets = targets.copy() ztargets['NUMOBS'] = np.zeros(n, dtype=np.int32) ztargets['Z'] = -1 * np.ones(n, dtype=np.float32) ztargets['ZWARN'] = -1 * np.ones(n, dtype=np.int32) ztargets['NUMOBS_MORE'] = np.maximum(0, calc_numobs(ztargets) - ztargets['NUMOBS']) mtl = ztargets.copy() ### mtl['NUMOBS_MORE'] = ztargets['NUMOBS_MORE'] mtl['PRIORITY'] = calc_priority(ztargets) #- If priority went to 0, then NUMOBS_MORE should also be 0 ii = (mtl['PRIORITY'] == 0) mtl['NUMOBS_MORE'][ii] = 0 #- remove extra zcat columns from join(targets, zcat) that are not needed #- for final MTL output for name in ['NUMOBS', 'Z', 'ZWARN']: mtl.remove_column(name) #- ELGs can be observed during gray time graylayer = np.zeros(n, dtype='i4') iselg = (mtl['DESI_TARGET'] & desi_mask.ELG) != 0 graylayer[iselg] = 1 mtl['GRAYLAYER'] = graylayer if trim: notdone = mtl['NUMOBS_MORE'] > 0 mtl = mtl[notdone] #- filtering can reset the fill_value, which is just wrong wrong wrong #- See https://github.com/astropy/astropy/issues/4707 #- and https://github.com/astropy/astropy/issues/4708 mtl['NUMOBS_MORE'].fill_value = -1 return mtl
class Pan_STARRS_Survey(surveycoord.SurveyCoord): """ A class to access all the catalogs hosted on the Vizier database. Inherits from SurveyCoord. This is a super class not meant for use by itself and instead meant to instantiate specific children classes like PAN-STARRS_Survey """ def __init__(self, coord, radius, **kwargs): surveycoord.SurveyCoord.__init__(self, coord, radius, **kwargs) self.Survey = "Pan_STARRS" def get_catalog(self, query_fields=None, release="dr2", table="stack"): """ Query a catalog in the VizieR database for photometry. Args: query_fields: list, optional A list of query fields to get. release: str, optional "dr1" or "dr2" (default: "dr2"). Data release version. table: str, optional "mean","stack" or "detection" (default: "stack"). The data table to search within. Returns: catalog: astropy.table.Table Contains all query results """ assert self.radius <= 0.5 * u.deg, "Cone serches have a maximum radius" #Validate table and release input _check_legal(table, release) url = "https://catalogs.mast.stsci.edu/api/v0.1/panstarrs/{:s}/{:s}.csv".format( release, table) if query_fields is None: query_fields = [ 'objID', 'raStack', 'decStack', 'objInfoFlag', 'qualityFlag' ] query_fields += [ '{:s}PSFmag'.format(band) for band in PanSTARRS_bands ] query_fields += [ '{:s}PSFmagErr'.format(band) for band in PanSTARRS_bands ] #Validate columns _check_columns(query_fields, table, release) data = {} data['ra'] = self.coord.ra.value data['dec'] = self.coord.dec.value data['radius'] = self.radius.to(u.deg).value data['columns'] = query_fields ret = requests.get(url, params=data) ret.raise_for_status() if len(ret.text) == 0: self.catalog = Table() self.catalog.meta['radius'] = self.radius self.catalog.meta['survey'] = self.survey # Validate self.validate_catalog() return self.catalog.copy() photom_catalog = Table.read(ret.text, format="ascii.csv") pdict = photom['Pan-STARRS'] photom_catalog = catalog_utils.clean_cat(photom_catalog, pdict) # self.catalog = catalog_utils.sort_by_separation(photom_catalog, self.coord, radec=('ra', 'dec'), add_sep=True) # Meta self.catalog.meta['radius'] = self.radius self.catalog.meta['survey'] = self.survey #Validate self.validate_catalog() #Return return self.catalog.copy() def get_cutout(self, imsize=30 * u.arcsec, filt="irg", output_size=None): """ Grab a color cutout (PNG) from Pan-STARRS Args: imsize (Quantity): Angular size of image desired filt (str): A string with the three filters to be used output_size (int): Output image size in pixels. Defaults to the original cutout size. Returns: PNG image, None (None for the header). """ assert len(filt) == 3, "Need three filters for a cutout." #Sort filters from red to blue filt = filt.lower( ) #Just in case the user is cheeky about the filter case. reffilt = "yzirg" idx = np.argsort([reffilt.find(f) for f in filt]) newfilt = "" for i in idx: newfilt += filt[i] #Get image url url = _get_url(self.coord, imsize=imsize, filt=newfilt, output_size=output_size, color=True, imgformat='png') self.cutout = images.grab_from_url(url) self.cutout_size = imsize return self.cutout.copy(), def get_image(self, imsize=30 * u.arcsec, filt="i", timeout=120): """ Grab a fits image from Pan-STARRS in a specific band. Args: imsize (Quantity): Angular size of the image desired filt (str): One of 'g','r','i','z','y' (default: 'i') timeout (int): Number of seconds to timout the query (default: 120 s) Returns: hdu: fits header data unit for the downloaded image """ assert len( filt ) == 1 and filt in "grizy", "Filter name must be one of 'g','r','i','z','y'" url = _get_url(self.coord, imsize=imsize, filt=filt, imgformat='fits')[0] imagedat = fits.open( astroutils.data.download_file(url, cache=True, show_progress=False, timeout=timeout))[0] return imagedat
def injectprev(self,fmll,presetll=None): # # determine if there are any pre-initialized lines (from previous run) and set those free parameters # make a unique ID for each line fmllcode = np.empty(len(fmll),dtype=object) for nnn,ill in enumerate(fmll): fmllcode[nnn] = "".join( [ str(ill['WL']), str(ill['GFLOG']), ill['CODE'], # str(ill['E']),str(ill['EP']), str(ill['XJ']),str(ill['XJP']), str(ill['LABEL']),str(ill['LABELP']), # ill['REF'], # str(ill['ISO1']),str(ill['X1']), # str(ill['ISO2']),str(ill['X2']), str(ill['ISO1']), str(ill['ISO2']), ill['OTHER'] ] ).replace(" ","") # Read previous table: LINE INFO, DWL, DGFLOG, DGAMMA (will figure out which GAMMA after the fact) if presetll == None: initlines = '/n/conroyfs1/pac/FAL/data/LL/SL_pars4.h5' ilines = Table(np.array(h5py.File(initlines,'r')['data'])) else: initlines = presetll ilines_i = h5py.File(initlines,'r')['ll'] ilines_spaces = np.linspace(0,ilines_i.len(),10,dtype=int) for ii in range(len(ilines_spaces)-1): ilines_ii = np.array(ilines_i[ilines_spaces[ii]:ilines_spaces[ii+1]-1]) selind = np.in1d(ilines_ii['WL'],fmll['WL']) ilines_iii = Table(ilines_ii[selind]) if ii == 0: ilines = ilines_iii.copy() else: if len(ilines_iii) > 0: ilines = vstack([ilines,ilines_iii]) # make unique ID for lines in preset linelist ilines.sort('WL') ilines = ilines[ilines['DWL'] != 0.0] ilines['UNIQ_ID'] = np.empty(len(ilines),dtype=object) for nnn,ill in enumerate(ilines): ilines['UNIQ_ID'][nnn] = "".join( [ str(ill['WL']), str(ill['GFLOG']), ill['CODE'], # str(ill['E']),str(ill['EP']), str(ill['XJ']),str(ill['XJP']), str(ill['LABEL']),str(ill['LABELP']), # ill['REF'], # str(ill['ISO1']),str(ill['X1']), # str(ill['ISO2']),str(ill['X2']), str(ill['ISO1']), str(ill['ISO2']), ill['OTHER'][1:] ] ).replace(" ","") # cycle through line list and find matches numpreset = 0 for ii,fmlc in enumerate(fmllcode): cond_intl = np.in1d(ilines['UNIQ_ID'],fmlc,assume_unique=True) if any(cond_intl): numpreset = numpreset + 1 # print("Pro: {0} --> Setting Previous Pars for WL = {1:7.4f}".format(self.ID,float(fmll['WL'][ii]))) fmll['DWL'][ii] = float('{0:6.4f}'.format(float(ilines['DWL'][cond_intl]))) fmll['DGFLOG'][ii] = float('{0:6.4f}'.format(float(ilines['DGFLOG'][cond_intl]))) fmll['DGAMMAW'][ii] = float('{0:6.4f}'.format(float(ilines['DGAMMAW'][cond_intl]))) fmll['DGAMMAR'][ii] = float('{0:6.4f}'.format(float(ilines['DGAMMAR'][cond_intl]))) fmll['DGAMMAS'][ii] = float('{0:6.4f}'.format(float(ilines['DGAMMAS'][cond_intl]))) print("Pro: {0} --> Setting Previous Pars for {1} lines from {2}".format(self.ID,numpreset,initlines)) # clean up some memory del ilines del ilines_i del ilines_ii del ilines_iii return fmll
def get_modern_data(manvr, dwell, starcheck): catalog = Table(starcheck['cat']) catalog.sort('idx') # Filter the catalog to be just acquisition stars catalog = catalog[(catalog['type'] == 'ACQ') | (catalog['type'] == 'BOT')] slot_for_pos = [cat_row['slot'] for cat_row in catalog] pos_for_slot = dict([(slot, idx) for idx, slot in enumerate(slot_for_pos)]) # Also, save out the starcheck index for each slot for later index_for_slot = dict([(cat_row['slot'], cat_row['idx']) for cat_row in catalog]) # Get telemetry msids = [ 'AOACASEQ', 'AOACQSUC', 'AOFREACQ', 'AOFWAIT', 'AOREPEAT', 'AOACSTAT', 'AOACHIBK', 'AOFSTAR', 'AOFATTMD', 'AOACPRGS', 'AOATUPST', 'AONSTARS', 'AOPCADMD', 'AORFSTR1', 'AORFSTR2', 'AOATTQT1', 'AOATTQT2', 'AOATTQT3', 'AOATTQT4' ] per_slot = [ 'AOACQID', 'AOACFCT', 'AOIMAGE', 'AOACMAG', 'AOACYAN', 'AOACZAN', 'AOACICC', 'AOACIDP', 'AOACIIR', 'AOACIMS', 'AOACIQB', 'AOACISP' ] slot_msids = [ field + '%s' % slot for field in per_slot for slot in range(0, 8) ] start_time = DateTime(manvr.acq_start).secs stop_time = DateTime(dwell.start).secs + 100 raw_eng_data = fetch.MSIDset(msids + slot_msids, start_time, stop_time, filter_bad=True) eng_data = Table([raw_eng_data[col].vals for col in msids], names=msids) for field in slot_msids: eng_data.add_column(Column(name=field, data=raw_eng_data[field].vals)) times = Table([raw_eng_data['AOACASEQ'].times], names=['time']) if not len(eng_data['AOACASEQ']): raise ValueError("No telemetry for obsid {}".format(manvr.get_obsid())) # Estimate the offsets from the expected catalog positions dy, dz, star_info = _deltas_vs_obc_quat(eng_data, times['time'], catalog) # And add the deltas to the table for slot in range(0, 8): if slot not in dy: continue eng_data.add_column( Column(name='dy{}'.format(slot), data=dy[slot].data)) eng_data.add_column( Column(name='dz{}'.format(slot), data=dz[slot].data)) cat_entry = catalog[catalog['slot'] == slot][0] dmag = eng_data['AOACMAG{}'.format(slot)] - cat_entry['mag'] eng_data.add_column(Column(name='dmag{}'.format(slot), data=dmag.data)) # Get the one-shot delta quaternion and the dot product of the deltas delta_quat, dot_q = get_delta_quat(eng_data, times['time'], manvr) one_shot_length = np.degrees(2 * np.arccos(dot_q)) one_shot_length = np.min([one_shot_length, 360 - one_shot_length]) one_shot_length = one_shot_length * 3600 # Update a copy of the telemetry structure with quaternions # corrected by the one-shot delta corr_eng_data = eng_data.copy() uncorr_times = (times['time'] < DateTime(manvr.guide_start).secs + 1.0) q_orig = Quat(q=np.array([ eng_data[uncorr_times]['AOATTQT1'], eng_data[uncorr_times]['AOATTQT2'], eng_data[uncorr_times]['AOATTQT3'], eng_data[uncorr_times]['AOATTQT4'] ]).transpose()) q_corr = q_mult(delta_quat.q, q_orig.q) corr_eng_data['AOATTQT1'][uncorr_times] = q_corr.q.transpose()[0] corr_eng_data['AOATTQT2'][uncorr_times] = q_corr.q.transpose()[1] corr_eng_data['AOATTQT3'][uncorr_times] = q_corr.q.transpose()[2] corr_eng_data['AOATTQT4'][uncorr_times] = q_corr.q.transpose()[3] corr_dy, corr_dz, si = _deltas_vs_obc_quat(corr_eng_data, times['time'], catalog) # delete the now-extra copy of the data del corr_eng_data # And add the corrected deltas to the table for slot in range(0, 8): if slot not in corr_dy: continue eng_data.add_column( Column(name='corr_dy{}'.format(slot), data=corr_dy[slot].data)) eng_data.add_column( Column(name='corr_dz{}'.format(slot), data=corr_dz[slot].data)) # Also add the acquisition id in a useful way for slot in range(0, 8): if slot not in pos_for_slot: continue eng_data.add_column( Column(name='POS_ACQID{}'.format(slot), data=eng_data['AOACQID{}'.format(pos_for_slot[slot])])) return eng_data, times['time'], one_shot_length, star_info
class TestPriorities(unittest.TestCase): def setUp(self): targdtype = [('DESI_TARGET', np.int64), ('BGS_TARGET', np.int64), ('MWS_TARGET', np.int64), ('PRIORITY_INIT', np.int64), ('NUMOBS_INIT', np.int64)] zdtype = [ ('Z', np.float32), ('ZWARN', np.float32), ('NUMOBS', np.float32), ] n = 3 self.targets = Table(np.zeros(n, dtype=targdtype)) self.targets['TARGETID'] = list(range(n)) self.zcat = Table(np.zeros(n, dtype=zdtype)) self.zcat['TARGETID'] = list(range(n)) def test_priorities(self): """Test that priorities are set correctly for both the main survey and SV. """ # ADM loop through once for SV and once for the main survey. for prefix in ["", "SV1_"]: t = self.targets.copy() z = self.zcat.copy() main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET'] for name in main_names: t.rename_column(name, prefix + name) # ADM retrieve the mask and column names for this survey flavor. colnames, masks, _ = main_cmx_or_sv(t) desi_target, bgs_target, mws_target = colnames desi_mask, bgs_mask, mws_mask = masks # - No targeting bits set is priority=0 self.assertTrue(np.all(calc_priority(t, z) == 0)) # - test QSO > (LRG_1PASS | LRG_2PASS) > ELG t[desi_target] = desi_mask.ELG self.assertTrue( np.all( calc_priority(t, z) == desi_mask.ELG.priorities['UNOBS'])) t[desi_target] |= desi_mask.LRG_1PASS self.assertTrue( np.all( calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS'])) t[desi_target] |= desi_mask.LRG_2PASS self.assertTrue( np.all( calc_priority(t, z) == desi_mask.LRG.priorities['UNOBS'])) t[desi_target] |= desi_mask.QSO self.assertTrue( np.all( calc_priority(t, z) == desi_mask.QSO.priorities['UNOBS'])) # - different states -> different priorities # - Done is Done, regardless of ZWARN. t[desi_target] = desi_mask.ELG t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], desi_mask.ELG.priorities['UNOBS']) self.assertEqual(p[1], desi_mask.ELG.priorities['DONE']) self.assertEqual(p[2], desi_mask.ELG.priorities['DONE']) # - BGS FAINT targets are never DONE, only MORE_ZGOOD. t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_FAINT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_FAINT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_FAINT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_FAINT.priorities['MORE_ZGOOD']) # BGS_FAINT: {UNOBS: 2000, MORE_ZWARN: 2000, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} # - BGS BRIGHT targets are never DONE, only MORE_ZGOOD. t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_BRIGHT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD']) # BGS_BRIGHT: {UNOBS: 2100, MORE_ZWARN: 2100, MORE_ZGOOD: 1000, DONE: 2, OBS: 1, DONOTOBSERVE: 0} # BGS targets are NEVER done even after 100 observations t[desi_target] = desi_mask.BGS_ANY t[bgs_target] = bgs_mask.BGS_BRIGHT t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 100, 100] z['ZWARN'] = [1, 1, 0] p = calc_priority(t, z) self.assertEqual(p[0], bgs_mask.BGS_BRIGHT.priorities['UNOBS']) self.assertEqual(p[1], bgs_mask.BGS_BRIGHT.priorities['MORE_ZWARN']) self.assertEqual(p[2], bgs_mask.BGS_BRIGHT.priorities['MORE_ZGOOD']) # BGS ZGOOD targets always have lower priority than MWS targets that # are not DONE. # ADM first discard N/S informational bits from bitmask as these # ADM should never trump the other bits. bgs_names = [ name for name in bgs_mask.names() if 'NORTH' not in name and 'SOUTH' not in name ] mws_names = [ name for name in mws_mask.names() if 'NORTH' not in name and 'SOUTH' not in name ] lowest_bgs_priority_zgood = min( [bgs_mask[n].priorities['MORE_ZGOOD'] for n in bgs_names]) lowest_mws_priority_unobs = min( [mws_mask[n].priorities['UNOBS'] for n in mws_names]) lowest_mws_priority_zwarn = min( [mws_mask[n].priorities['MORE_ZWARN'] for n in mws_names]) lowest_mws_priority_zgood = min( [mws_mask[n].priorities['MORE_ZGOOD'] for n in mws_names]) lowest_mws_priority = min(lowest_mws_priority_unobs, lowest_mws_priority_zwarn, lowest_mws_priority_zgood) self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority) def test_bright_mask(self): t = self.targets z = self.zcat t['DESI_TARGET'][0] = desi_mask.ELG t['DESI_TARGET'][1] = desi_mask.ELG | desi_mask.NEAR_BRIGHT_OBJECT t['DESI_TARGET'][2] = desi_mask.ELG | desi_mask.IN_BRIGHT_OBJECT p = calc_priority(t, z) self.assertEqual( p[0], p[1], "NEAR_BRIGHT_OBJECT shouldn't impact priority but {} != {}".format( p[0], p[1])) self.assertEqual(p[2], -1, "IN_BRIGHT_OBJECT priority not -1") def test_mask_priorities(self): for mask in [desi_mask, bgs_mask, mws_mask]: for name in mask.names(): if name.startswith('STD') or name in [ 'BGS_ANY', 'MWS_ANY', 'SECONDARY_ANY', 'IN_BRIGHT_OBJECT', 'NEAR_BRIGHT_OBJECT', 'BRIGHT_OBJECT', 'SKY', 'SV', 'NO_TARGET' ]: self.assertEqual(mask[name].priorities, {}, 'mask.{} has priorities?'.format(name)) else: for state in obsmask.names(): self.assertIn( state, mask[name].priorities, '{} not in mask.{}.priorities'.format(state, name)) def test_cmx_priorities(self): """Test that priority calculation can handle commissioning files. """ t = self.targets.copy() z = self.zcat # ADM restructure the table to look like a commissioning table. t.rename_column('DESI_TARGET', 'CMX_TARGET') t.remove_column('BGS_TARGET') t.remove_column('MWS_TARGET') # - No targeting bits set is priority=0 self.assertTrue(np.all(calc_priority(t, z) == 0)) # ADM retrieve the cmx_mask. colnames, masks, _ = main_cmx_or_sv(t) cmx_mask = masks[0] # ADM test handling of unobserved SV0_BGS and SV0_MWS. for name in ["SV0_BGS", "SV0_MWS"]: t['CMX_TARGET'] = cmx_mask[name] self.assertTrue( np.all( calc_priority(t, z) == cmx_mask[name].priorities['UNOBS'])) # ADM done is Done, regardless of ZWARN. for name in ["SV0_BGS", "SV0_MWS"]: t['CMX_TARGET'] = cmx_mask[name] t["PRIORITY_INIT"], t["NUMOBS_INIT"] = initial_priority_numobs(t) z['NUMOBS'] = [0, 1, 1] z['ZWARN'] = [1, 1, 0] p = make_mtl(t, z)["PRIORITY"] self.assertEqual(p[0], cmx_mask[name].priorities['UNOBS']) self.assertEqual(p[1], cmx_mask[name].priorities['DONE']) self.assertEqual(p[2], cmx_mask[name].priorities['DONE']) # BGS ZGOOD targets always have lower priority than MWS targets that # are not DONE. lowest_bgs_priority_zgood = cmx_mask['SV0_BGS'].priorities[ 'MORE_ZGOOD'] lowest_mws_priority_unobs = cmx_mask['SV0_MWS'].priorities['UNOBS'] lowest_mws_priority_zwarn = cmx_mask['SV0_MWS'].priorities[ 'MORE_ZWARN'] lowest_mws_priority_zgood = cmx_mask['SV0_MWS'].priorities[ 'MORE_ZGOOD'] lowest_mws_priority = min(lowest_mws_priority_unobs, lowest_mws_priority_zwarn, lowest_mws_priority_zgood) self.assertLess(lowest_bgs_priority_zgood, lowest_mws_priority)
class TimeTable: masked = False def setup(self): # Initialize table self.table = Table(masked=self.masked) # Create column with mixed types np.random.seed(12345) self.table['i'] = np.arange(1000) self.table['a'] = np.random.random(1000) # float self.table['b'] = np.random.random(1000) > 0.5 # bool self.table['c'] = np.random.random((1000,10)) # 2d column self.table['d'] = np.random.choice(np.array(list(string.ascii_letters)),1000) self.np_table = np.array(self.table) self.extra_row = {'a':1.2, 'b':True, 'c':np.repeat(1, 10), 'd': 'Z'} self.extra_column = np.random.randint(0, 100, 1000) self.row_indices = np.where(self.table['a'] > 0.9)[0] self.table_grouped = self.table.group_by('d') # Another table for testing joining self.other_table = Table(masked=self.masked) self.other_table['i'] = np.arange(1,1000,3) self.other_table['f'] = np.random.random() self.other_table.sort('f') # Another table for testing hstack self.other_table_2 = Table(masked=self.masked) self.other_table_2['g'] = np.random.random(1000) self.other_table_2['h'] = np.random.random((1000, 10)) self.bool_mask = self.table['a'] > 0.6 def time_table_slice_bool(self): table_subset = self.table[self.bool_mask] def time_table_slice_int(self): table_subset = self.table[self.row_indices] def time_column_slice_bool(self): col_subset = self.table['a'][self.bool_mask] def time_column_slice_int(self): col_subset = self.table['a'][self.row_indices] def time_column_get(self): self.table['c'] def time_column_make_bool_mask(self): self.table['a'] > 0.6 def time_multi_column_get(self): self.table[('a','c')] def time_column_set(self): self.table['a'] = 0. def time_column_set_all(self): self.table['b'][:] = True def time_column_set_row_subset(self): self.table['b'][self.bool_mask] = True def time_column_set_row_subset_int(self): self.table['b'][self.row_indices] = True def time_row_get(self): self.table[300] def time_iter_row(self): for row in self.table: pass def time_read_rows(self): for row in self.table: tuple(row) def time_item_get_rowfirst(self): self.table[300]['b'] def time_item_get_colfirst(self): self.table['b'][300] def time_add_row(self): self.table.add_row(self.extra_row) time_add_row.number = 1 time_add_row.repeat = 1 def time_remove_row(self): self.table.remove_row(6) time_remove_row.number = 1 time_remove_row.repeat = 1 def time_remove_rows(self): self.table.remove_rows(self.row_indices) time_remove_rows.number = 1 time_remove_rows.repeat = 1 def time_add_column(self): self.table['e'] = self.extra_column time_add_column.number = 1 time_add_column.repeat = 1 def time_remove_column(self): self.table.remove_column('a') time_remove_column.number = 1 time_remove_column.repeat = 1 def time_init_from_np_array_no_copy(self): Table(self.np_table, copy=False) def time_init_from_np_array_copy(self): Table(self.np_table, copy=True) def time_copy_table(self): self.table.copy() def time_copy_column(self): self.table['a'].copy() def time_group(self): self.table.group_by('d') def time_aggregate(self): # Test aggregate with a function that supports reduceat self.table_grouped.groups.aggregate(np.sum) def time_aggregate_noreduceat(self): # Test aggregate with a function that doesn't support reduceat self.table_grouped.groups.aggregate(lambda x: np.sum(x)) def time_sort(self): self.table.sort('a') def time_join_inner(self): join(self.table, self.other_table, keys="i", join_type='inner') def time_join_outer(self): join(self.table, self.other_table, keys="i", join_type='outer') def time_hstack(self): hstack([self.table, self.other_table_2]) def time_vstack(self): vstack([self.table, self.table])
'band', 'bsens', 'selfcaliter', ]) tbl.write("fields_summary_table.ecsv", overwrite=True) shutil.copy( "fields_summary_table.ecsv", "/orange/adamginsburg/web/secure/ALMA-IMF/October2020Release/tables/") from latex_info import (latexdict, format_float, round_to_n, rounded, rounded_arr, strip_trailing_zeros, exp_to_tex) latexdict = latexdict.copy() ltbl = tbl.copy() ltbl['fields'] = [len(x) for x in tbl['fields']] # caption needs to be *before* preamble. #latexdict['caption'] = 'Continuum Source IDs and photometry' latexdict[ 'header_start'] = '\label{tab:selfcal_fields}' #\n\\footnotesize' latexdict[ 'preamble'] = '\caption{Selfcal Field Inclusion}\n\\resizebox{\\textwidth}{!}{' latexdict['col_align'] = 'l' * len(ltbl.columns) latexdict['tabletype'] = 'table*' latexdict['tablefoot'] = ( "}\par\n" "Number of fields included in each self-calibration entry") ltbl.write(
def get_modern_data(manvr, dwell, starcheck): catalog = Table(starcheck['cat']) catalog.sort('idx') # Filter the catalog to be just acquisition stars catalog = catalog[(catalog['type'] == 'ACQ') | (catalog['type'] == 'BOT')] slot_for_pos = [cat_row['slot'] for cat_row in catalog] pos_for_slot = dict([(slot, idx) for idx, slot in enumerate(slot_for_pos)]) # Also, save out the starcheck index for each slot for later index_for_slot = dict([(cat_row['slot'], cat_row['idx']) for cat_row in catalog]) # Get telemetry msids = ['AOACASEQ', 'AOACQSUC', 'AOFREACQ', 'AOFWAIT', 'AOREPEAT', 'AOACSTAT', 'AOACHIBK', 'AOFSTAR', 'AOFATTMD', 'AOACPRGS', 'AOATUPST', 'AONSTARS', 'AOPCADMD', 'AORFSTR1', 'AORFSTR2', 'AOATTQT1', 'AOATTQT2', 'AOATTQT3', 'AOATTQT4'] per_slot = ['AOACQID', 'AOACFCT', 'AOIMAGE', 'AOACMAG', 'AOACYAN', 'AOACZAN', 'AOACICC', 'AOACIDP', 'AOACIIR', 'AOACIMS', 'AOACIQB', 'AOACISP'] slot_msids = [field + '%s' % slot for field in per_slot for slot in range(0, 8)] start_time = DateTime(manvr.acq_start).secs stop_time = DateTime(dwell.start).secs + 100 raw_eng_data = fetch.MSIDset(msids + slot_msids, start_time, stop_time, filter_bad=True) eng_data = Table([raw_eng_data[col].vals for col in msids], names=msids) for field in slot_msids: eng_data.add_column( Column( name=field, data=raw_eng_data[field].vals)) times = Table([raw_eng_data['AOACASEQ'].times], names=['time']) if not len(eng_data['AOACASEQ']): raise ValueError("No telemetry for obsid {}".format(manvr.get_obsid())) # Estimate the offsets from the expected catalog positions dy, dz, star_info = _deltas_vs_obc_quat(eng_data, times['time'], catalog) # And add the deltas to the table for slot in range(0, 8): if slot not in dy: continue eng_data.add_column(Column(name='dy{}'.format(slot), data=dy[slot].data)) eng_data.add_column(Column(name='dz{}'.format(slot), data=dz[slot].data)) cat_entry = catalog[catalog['slot'] == slot][0] dmag = eng_data['AOACMAG{}'.format(slot)] - cat_entry['mag'] eng_data.add_column(Column(name='dmag{}'.format(slot), data=dmag.data)) # Get the one-shot delta quaternion and the dot product of the deltas delta_quat, dot_q = get_delta_quat(eng_data, times['time'], manvr) one_shot_length = np.degrees(2 * np.arccos(dot_q)) one_shot_length = np.min([one_shot_length, 360 - one_shot_length]) one_shot_length = one_shot_length * 3600 # Update a copy of the telemetry structure with quaternions # corrected by the one-shot delta corr_eng_data = eng_data.copy() uncorr_times = (times['time'] < DateTime(manvr.guide_start).secs + 1.0) q_orig = Quat(np.array([eng_data[uncorr_times]['AOATTQT1'], eng_data[uncorr_times]['AOATTQT2'], eng_data[uncorr_times]['AOATTQT3'], eng_data[uncorr_times]['AOATTQT4']]).transpose()) q_corr = q_mult(delta_quat.q, q_orig.q) corr_eng_data['AOATTQT1'][uncorr_times] = q_corr.q.transpose()[0] corr_eng_data['AOATTQT2'][uncorr_times] = q_corr.q.transpose()[1] corr_eng_data['AOATTQT3'][uncorr_times] = q_corr.q.transpose()[2] corr_eng_data['AOATTQT4'][uncorr_times] = q_corr.q.transpose()[3] corr_dy, corr_dz, si = _deltas_vs_obc_quat(corr_eng_data, times['time'], catalog) # delete the now-extra copy of the data del corr_eng_data # And add the corrected deltas to the table for slot in range(0, 8): if slot not in corr_dy: continue eng_data.add_column(Column(name='corr_dy{}'.format(slot), data=corr_dy[slot].data)) eng_data.add_column(Column(name='corr_dz{}'.format(slot), data=corr_dz[slot].data)) # Also add the acquisition id in a useful way for slot in range(0, 8): if slot not in pos_for_slot: continue eng_data.add_column( Column( name='POS_ACQID{}'.format(slot), data=eng_data['AOACQID{}'.format(pos_for_slot[slot])])) return eng_data, times['time'], one_shot_length, star_info
def run(cfg): """ Run hugs pipeline using SExtractor for the final detection and photometry. Parameters ---------- cfg : hugs_pipe.Config Configuration object which stores all params as well as the exposure object. Returns ------- results : lsst.pipe.base.Struct Object containing results: results.all_detections : catalog of all detections results.sources : catalog of sources we are keeping results.exp : exposure object for this run results.exp_clean : cleaned exposure object for this run results.success : boolean flag of run status """ assert cfg.tract and cfg.patch, 'No patch id given!' cfg.timer # start timer ############################################################ # Get masked image and check if we have enough good data ############################################################ try: mi = cfg.exp[cfg.band_detect].getMaskedImage() mask = mi.getMask() if cfg.exp.patch_meta.good_data_frac < cfg.min_good_data_frac: cfg.logger.warning('***** not enough data!!! ****') results = _null_return(cfg) return results ############################################################ # Image thesholding at low and high thresholds. In both # cases, the image is smoothed at the psf scale. ############################################################ mi_smooth = imtools.smooth_gauss(mi, cfg.psf_sigma) cfg.logger.info('performing low threshold at ' '{} sigma'.format(cfg.thresh_low['thresh'])) fpset_low = prim.image_threshold(mi_smooth, mask=mask, plane_name='THRESH_LOW', **cfg.thresh_low) cfg.logger.info('performing high threshold at ' '{} sigma'.format(cfg.thresh_high['thresh'])) fpset_high = prim.image_threshold(mi_smooth, mask=mask, plane_name='THRESH_HIGH', **cfg.thresh_high) ############################################################ # Get "cleaned" image, with noise replacement ############################################################ cfg.logger.info('generating cleaned exposure') exp_clean = prim.clean(cfg.exp[cfg.band_detect], fpset_low, **cfg.clean) mi_clean = exp_clean.getMaskedImage() mask_clean = mi_clean.getMask() ############################################################ # Detect sources and measure props with SExtractor ############################################################ cfg.logger.info('detecting in {}-band'.format(cfg.band_detect)) label = '{}-{}-{}'.format(cfg.tract, cfg.patch[0], cfg.patch[-1]) cfg.logger.info('cleaning non-detection bands') replace = cfg.exp.get_mask_array(cfg.band_detect) for band in cfg.bands: if band != cfg.band_detect: mi_band = cfg.exp[band].getMaskedImage() noise_array = utils.make_noise_image(mi_band, cfg.rng) mi_band.getImage().getArray()[replace] = noise_array[replace] sources = Table() for band in cfg.bands: cfg.logger.info('measuring in {}-band'.format(band)) dual_exp = None if band == cfg.band_detect else cfg.exp[band] sources_band = prim.detect_sources( exp_clean, cfg.sex_config, cfg.sex_io_dir, label=label, dual_exp=dual_exp, delete_created_files=cfg.delete_created_files, original_fn=cfg.exp.fn[cfg.band_detect]) if len(sources_band) > 0: sources = hstack([sources, sources_band]) else: cfg.logger.warn('**** no sources found by sextractor ****') results = _null_return(cfg, exp_clean) return results ############################################################ # Verify detections in other bands using SExtractor ############################################################ all_detections = sources.copy() for band in cfg.band_verify: cfg.logger.info('verifying dection in {}-band'.format(band)) sources_verify = prim.detect_sources( cfg.exp[band], cfg.sex_config, cfg.sex_io_dir, label=label, delete_created_files=cfg.delete_created_files, original_fn=cfg.exp.fn[cfg.band_detect]) if len(sources_verify) > 0: match_masks, _ = xmatch(sources, sources_verify, max_sep=cfg.verify_max_sep) txt = 'cuts: {} out of {} objects detected in {}-band'.format( len(match_masks[0]), len(sources), band) cfg.logger.info(txt) if len(match_masks[0]) == 0: cfg.logger.warn('**** no matched sources with ' + band + ' ****') results = _null_return(cfg, exp_clean) return results sources = sources[match_masks[0]] else: cfg.logger.warn('**** no sources detected in ' + band + ' ****') results = _null_return(cfg, exp_clean) return results mask_fracs = utils.calc_mask_bit_fracs(exp_clean) cfg.exp.patch_meta.cleaned_frac = mask_fracs['cleaned_frac'] cfg.exp.patch_meta.bright_obj_frac = mask_fracs['bright_object_frac'] cfg.logger.info('task completed in {:.2f} min'.format(cfg.timer)) results = Struct(all_detections=all_detections, sources=sources, hugs_exp=cfg.exp, exp_clean=exp_clean, success=True, synths=cfg.exp.synths) cfg.reset_mask_planes() return results except Exception as e: cfg.logger.critical('tract - patch {} - {} failed: {}'.format( cfg.tract, cfg.patch, e)) results = _null_return(cfg) return results
class TestMTL(unittest.TestCase): def setUp(self): self.targets = Table() self.types = np.array(['ELG', 'LRG', 'QSO', 'QSO', 'ELG']) self.priorities = [Mx[t].priorities['UNOBS'] for t in self.types] self.post_prio = [Mx[t].priorities['MORE_ZGOOD'] for t in self.types] self.post_prio[0] = 2 # ELG self.post_prio[1] = 2 # LRG...all one-pass self.post_prio[2] = 2 # lowz QSO self.targets['DESI_TARGET'] = [Mx[t].mask for t in self.types] self.targets['BGS_TARGET'] = np.zeros(len(self.types), dtype=np.int64) self.targets['MWS_TARGET'] = np.zeros(len(self.types), dtype=np.int64) n = len(self.targets) self.targets['ZFLUX'] = 10**((22.5 - np.linspace(20, 22, n)) / 2.5) self.targets['TARGETID'] = list(range(n)) # ADM determine the initial PRIORITY and NUMOBS. pinit, ninit = initial_priority_numobs(self.targets) self.targets["PRIORITY_INIT"] = pinit self.targets["NUMOBS_INIT"] = ninit # - reverse the order for zcat to make sure joins work self.zcat = Table() self.zcat['TARGETID'] = self.targets['TARGETID'][-2::-1] self.zcat['Z'] = [2.5, 1.0, 0.5, 1.0] self.zcat['ZWARN'] = [0, 0, 0, 0] self.zcat['NUMOBS'] = [1, 1, 1, 1] self.zcat['SPECTYPE'] = ['QSO', 'QSO', 'GALAXY', 'GALAXY'] def reset_targets(self, prefix): """Add prefix to TARGET columns""" t = self.targets.copy() main_names = ['DESI_TARGET', 'BGS_TARGET', 'MWS_TARGET'] if prefix == 'CMX': # ADM restructure the table to look like a commissioning table. t.rename_column('DESI_TARGET', 'CMX_TARGET') t.remove_column('BGS_TARGET') t.remove_column('MWS_TARGET') else: for name in main_names: t.rename_column(name, prefix + name) # if prefix == "SV1_": # ADM change any occurences of LRG_2PASS to just LRG. # ADM technically not needed as 2PASS no longer exists. # ii = t["SV1_DESI_TARGET"] == Mx.LRG_2PASS # t["SV1_DESI_TARGET"][ii] = MxSV.LRG return t def test_mtl(self): """Test output from MTL has the correct column names. """ # ADM loop through once each for the main survey, commissioning and SV. for prefix in ["", "CMX_", "SV1_"]: t = self.reset_targets(prefix) mtl = make_mtl(t, "BRIGHT|GRAY|DARK") goodkeys = sorted( set(t.dtype.names) | set(['NUMOBS_MORE', 'PRIORITY', 'OBSCONDITIONS'])) mtlkeys = sorted(mtl.dtype.names) self.assertEqual(mtlkeys, goodkeys) def test_numobs(self): """Test priorities, numobs and obsconditions are set correctly with no zcat. """ # ADM loop through once for SV and once for the main survey. for prefix in ["", "SV1_"]: t = self.reset_targets(prefix) mtl = make_mtl(t, "GRAY|DARK") mtl.sort(keys='TARGETID') self.assertTrue(np.all(mtl['NUMOBS_MORE'] == [1, 1, 4, 4, 1])) self.assertTrue(np.all(mtl['PRIORITY'] == self.priorities)) # - Check that ELGs can be observed in gray conditions but not others iselg = (self.types == 'ELG') self.assertTrue( np.all( (mtl['OBSCONDITIONS'][iselg] & obsconditions.GRAY) != 0)) self.assertTrue( np.all((mtl['OBSCONDITIONS'][~iselg] & obsconditions.GRAY) == 0)) def test_zcat(self): """Test priorities, numobs and obsconditions are set correctly after zcat. """ # ADM loop through once for SV and once for the main survey. for prefix in ["", "SV1_"]: t = self.reset_targets(prefix) mtl = make_mtl(t, "DARK|GRAY", zcat=self.zcat, trim=False) mtl.sort(keys='TARGETID') pp = self.post_prio.copy() nom = [0, 0, 0, 3, 1] # ADM in SV, all quasars get all observations. if prefix == "SV1_": pp[2], nom[2] = pp[3], nom[3] self.assertTrue(np.all(mtl['PRIORITY'] == pp)) self.assertTrue(np.all(mtl['NUMOBS_MORE'] == nom)) # - change one target to a SAFE (BADSKY) target and confirm priority=0 not 1 t[prefix + 'DESI_TARGET'][0] = Mx.BAD_SKY mtl = make_mtl(t, "DARK|GRAY", zcat=self.zcat, trim=False) mtl.sort(keys='TARGETID') self.assertEqual(mtl['PRIORITY'][0], 0) def test_mtl_io(self): """Test MTL correctly handles masked NUMOBS quantities. """ # ADM loop through once for SV and once for the main survey. for prefix in ["", "SV1_"]: t = self.reset_targets(prefix) mtl = make_mtl(t, "BRIGHT", zcat=self.zcat, trim=True) testfile = 'test-aszqweladfqwezceas.fits' mtl.write(testfile, overwrite=True) x = mtl.read(testfile) os.remove(testfile) if x.masked: self.assertTrue( np.all(mtl['NUMOBS_MORE'].mask == x['NUMOBS_MORE'].mask))
def run(cfg, reset_mask_planes=True): """ Run hugs pipeline using SExtractor for the final detection and photometry. Parameters ---------- cfg : hugs_pipe.Config Configuration object which stores all params as well as the exposure object. Returns ------- results : lsst.pipe.base.Struct Object containing results: results.all_detections : catalog of all detections results.sources : catalog of sources we are keeping results.exp : exposure object for this run results.exp_clean : cleaned exposure object for this run results.success : boolean flag of run status """ assert cfg.tract and cfg.patch, 'No patch id given!' cfg.timer # start timer ############################################################ # Get masked image and check if we have enough good data ############################################################ mi = cfg.exp[cfg.band_detect].getMaskedImage() mask = mi.getMask() stat_task = get_clipped_sig_task() if cfg.exp.patch_meta.good_data_frac < cfg.min_good_data_frac: cfg.logger.warning('***** not enough data!!! ****') results = _null_return(cfg) return results ############################################################ # Image thesholding at low and high thresholds. In both # cases, the image is smoothed at the psf scale. ############################################################ #mi_smooth = imtools.smooth_gauss(mi, cfg.psf_sigma) stats = stat_task.run(mi) flux_th = 10**(0.4 * (zpt - cfg.thresh_low['thresh'])) * pixscale**2 cfg.thresh_low['thresh'] = flux_th / stats.stdev cfg.logger.info('performing low threshold at ' '{:.2f} sigma'.format(cfg.thresh_low['thresh'])) fpset_low = prim.image_threshold( mi, mask=mask, plane_name='THRESH_LOW', **cfg.thresh_low) flux_th = 10**(0.4 * (zpt - cfg.thresh_high['thresh'])) * pixscale**2 cfg.thresh_high['thresh'] = flux_th / stats.stdev cfg.logger.info('performing high threshold at ' '{:.2f} sigma'.format(cfg.thresh_high['thresh'])) fpset_high = prim.image_threshold( mi, mask=mask, plane_name='THRESH_HIGH', **cfg.thresh_high) ############################################################ # Get "cleaned" image, with noise replacement ############################################################ cfg.logger.info('generating cleaned exposure') exp_clean = prim.clean(cfg.exp[cfg.band_detect], fpset_low, **cfg.clean) mi_clean = exp_clean.getMaskedImage() mask_clean = mi_clean.getMask() ############################################################ # Detect sources and measure props with SExtractor ############################################################ cfg.logger.info('detecting in {}-band'.format(cfg.band_detect)) label = '{}-{}-{}'.format(cfg.tract, cfg.patch[0], cfg.patch[-1]) cfg.logger.info('cleaning non-detection bands') replace = cfg.exp.get_mask_array(cfg.band_detect) for band in cfg.bands: if band!=cfg.band_detect: mi_band = cfg.exp[band].getMaskedImage() noise_array = utils.make_noise_image(mi_band, cfg.rng) mi_band.getImage().getArray()[replace] = noise_array[replace] sources = Table() for band in cfg.bands: cfg.logger.info('measuring in {}-band'.format(band)) dual_exp = None if band==cfg.band_detect else cfg.exp[band] sources_band = prim.detect_sources( exp_clean, cfg.sex_config, cfg.sex_io_dir, label=label, dual_exp=dual_exp, delete_created_files=cfg.delete_created_files, original_fn=cfg.exp.fn[cfg.band_detect]) if len(sources_band)>0: sources = hstack([sources, sources_band]) else: cfg.logger.warn('**** no sources found by sextractor ****') results = _null_return(cfg, exp_clean) return results ############################################################ # Verify detections in other bands using SExtractor ############################################################ all_detections = sources.copy() for band in cfg.band_verify: cfg.logger.info('verifying dection in {}-band'.format(band)) sources_verify = prim.detect_sources( cfg.exp[band], cfg.sex_config, cfg.sex_io_dir, label=label, delete_created_files=cfg.delete_created_files, original_fn=cfg.exp.fn[band]) if len(sources_verify)>0: match_masks, _ = xmatch( sources, sources_verify, max_sep=cfg.verify_max_sep) txt = 'cuts: {} out of {} objects detected in {}-band'.format( len(match_masks[0]), len(sources), band) cfg.logger.info(txt) if len(match_masks[0])==0: cfg.logger.warn('**** no matched sources with '+band+' ****') results = _null_return(cfg, exp_clean) return results sources = sources[match_masks[0]] else: cfg.logger.warn('**** no sources detected in '+band+' ****') results = _null_return(cfg, exp_clean) return results mask_fracs = utils.calc_mask_bit_fracs(exp_clean) cfg.exp.patch_meta.cleaned_frac = mask_fracs['cleaned_frac'] cfg.exp.patch_meta.bright_obj_frac = mask_fracs['bright_object_frac'] cfg.logger.info('task completed in {:.2f} min'.format(cfg.timer)) results = Struct(all_detections=all_detections, sources=sources, exp=cfg.exp, exp_clean=exp_clean, success=True) if reset_mask_planes: cfg.reset_mask_planes() return results
""" Stellar Fits """ basename='{0}.{1}'.format('pickles', i+1) stellar = Table.read('{0}/{1}.zout'.format(folder, basename), format='ascii.commented_header') star_best = np.zeros(len(stellar), dtype='S6') star_best[stellar['temp_1'] >= -90] = st_names[stellar['temp_1']-1][stellar['temp_1'] >= -90] schi = stellar['chi_1']/(stellar['nfilt']-1) schi[stellar['chi_1'] == -99.] = -99. sub_cat['chi_r_stellar'] = schi sub_cat['stellar_type'] = star_best if i == 0: full_cat = Table.copy(sub_cat) else: full_cat = vstack([full_cat, sub_cat]) bar.update() folder = '{0}/full'.format(pipe_params.working_folder) path = '{0}/photoz_all_merged.fits'.format(folder) if os.path.isfile(path): os.remove(path) full_cat.write(path, format='fits') chis = np.array([full_cat['chi_r_eazy'], full_cat['chi_r_cosmos'], full_cat['chi_r_atlas']]) chi_best = np.min(chis, axis=0) hdf_hb.close()
def from_parsec(file_name, log_age=None, zini=None, num_filters=None): """ Read isochrone generated from the `PARSEC website <http://stev.oapd.inaf.it/cgi-bin/cmd>`_. If more than one age and/or metallicity is included in the file, you must provide the log_age and/or zini parameters. Parameters ---------- file_name : str Isochrone file name. log_age : float, optional Log of age in years. You must provided this parameter if there is more than one age in the isochrone file. Note this function does not interpolate ages, so it must be included in the file. zini : float, optional Initial metal fraction. You must provided this parameter if there is more than one metallicity in the isochrone file. Note this function does not interpolate metallicity, so it must be included in the file. num_filters : int, optional Number of filters included in the isochrone file. If None, will assume the last non-filter parameter is `mbolmag`. Returns ------- iso : `artpop.stars.Isochrone` PARSEC Isochrone object. """ if os.path.isfile(file_name): file = open(file_name, 'r') lines = file.readlines() file.close() for l in lines: if 'Zini' in l.split()[1]: names = l.split()[1:] break data = np.loadtxt(file_name) parsec = Table(data, names=names) else: raise Exception(f'{file_name} does not exist.') isochrone_full = parsec.copy() if log_age is not None: age_cut = np.abs(parsec['logAge'] - log_age) < 1e-5 if age_cut.sum() < 1: raise Exception(f'log_age = {log_age} not found.') parsec = parsec[age_cut] if zini is not None: zini_cut = np.abs(parsec['Zini'] - zini) < 1e-8 if zini_cut.sum() < 1: raise Exception(f'Zini = {zini} not found.') parsec = parsec[zini_cut] if num_filters is None: filt_idx = np.argwhere(np.array(names) == 'mbolmag')[0][0] + 1 else: filt_idx = len(names) - num_filters iso = Isochrone(mini=parsec['Mini'], mact=parsec['Mass'], mags=parsec[names[filt_idx:]], log_L=parsec['logL'], log_Teff=parsec['logTe']) iso.isochrone_full = isochrone_full return iso
def run_final_calibrations(self, initial_priors='parametric'): self.generate_evolution_tables() output_names = [ 'calib coefs', 'fit variances', 'wavelengths', 'pixels' ] all_output_names = [ 'calib coefs', 'fit variances', 'wavelengths', 'pixels', 'linelist' ] mock_spec_w, mock_spec_f = self.mock_spec_w, self.mock_spec_f if not self.do_fine_calib: print( "There doesn't seem to be a fine calibration defined. Using the supplied coarse calibs" ) select_lines = True dev_allowance = 1. devs = 2. using_defaults = False if initial_priors == 'defaults': if self.default_calibration_coefs is None: print( "Couldn't find the default calibration coefficients, so using a parametrization of the coarse coefs" ) initial_coef_table = Table( self.get_parametricfits_of(caltype='coarse')) else: need_to_parametrize = False initial_coef_table = Table(self.default_calibration_coefs) for fib in self.instrument.full_fibs[self.camera]: if fib not in initial_coef_table.colnames: need_to_parametrize = True break if need_to_parametrize: paramd_table = Table( self.get_parametricfits_of(caltype='default')) for fib in self.instrument.full_fibs[self.camera]: if fib not in initial_coef_table.colnames: initial_coef_table[fib] = paramd_table[fib] using_defaults = True elif initial_priors == 'medians': initial_coef_table = Table( self.get_medianfits_of(self.coarse_calibration_coefs)) else: initial_coef_table = Table( self.get_parametricfits_of(caltype='coarse')) for pairnum, filnums in self.pairings.items(): if pairnum > 0: coarse_table_differences = self.evolution_in_coarse_coefs[ pairnum] for column in coarse_table_differences.colnames: initial_coef_table[column] = initial_coef_table[ column] + coarse_table_differences[column] ## HACK!! if pairnum == 0 and self.camera == 'r': continue ## END HACK! filenum = filnums[self.filenum_ind] data = Table(self.fine_calibrations[filenum].data) linelist = self.selected_lines effective_iteration = pairnum #np.max([pairnum,int(using_defaults)]) if effective_iteration == 0: user_input = 'some' elif effective_iteration == 1: user_input = 'minimal' elif effective_iteration > 1: # and devs < dev_allowance: user_input = 'single' #'none' hand_fit_subset = [] cam = self.camera if user_input == 'all': hand_fit_subset = list(initial_coef_table.colnames) elif user_input in ['some', 'minimal', 'single']: if cam == 'r': # specific_set = [cam + '101', cam + '416'] specific_set = [ cam + '101', cam + '816', cam + '416', cam + '501' ] else: specific_set = [ cam + '116', cam + '801', cam + '516', cam + '401' ] for i, fib in enumerate(specific_set): outfib = ensure_match(fib, data.colnames, hand_fit_subset, cam) hand_fit_subset.append(outfib) if user_input == 'some': seed = int(filenum) np.random.seed(seed) randfibs = [ '{:02d}'.format(x) for x in np.random.randint(1, 16, 4) ] for tetn, fibn in zip([2, 3, 6, 7], randfibs): fib = '{}{}{}'.format(cam, tetn, fibn) outfib = ensure_match(fib, data.colnames, hand_fit_subset, cam) hand_fit_subset.append(outfib) elif user_input == 'single': hand_fit_subset = hand_fit_subset[:1] else: pass # hand_fit_subset = np.asarray(hand_fit_subset) ##HACK! # if pairnum == 0 and self.camera=='r': # altered_coef_table = initial_coef_table.copy() # hand_fit_subset = np.asarray(['r101','r816','r416','r501','r210','r602','r715']) # altered_coef_table = {}#initial_coef_table.copy() # altered_coef_table['r101'] = [5071.8187300612035, 0.9930979838081959, -5.769775729541421e-06, # 1.6219475654346627e-08, -1.060536238512127e-11, 2.027614894968671e-15] # # altered_coef_table['r816'] = [5064.941399949152, 0.9887048293667995, 4.829092351762018e-06, # 5.280389577236655e-09, -5.618906483279477e-12, 1.1981097537960155e-15] # # altered_coef_table['r416'] = [4966.43139830805, 0.9939388787553181, 5.244911711992524e-06, # 1.2291548669411035e-09, - 2.0296595329597448e-12, 2.9050877132565224e-16] # # altered_coef_table['r501'] = [4965.341783218052, 0.9873531089008049, 2.4560812264245633e-05, # -2.0293237635901715e-08, 8.081202360788054e-12, -1.397383927434781e-15] # # altered_coef_table['r210'] = [5009.879532180203, 0.986418938077269, # 2.1117286784979934e-05, - 1.612921025968839e-08, 6.307242237439978e-12, # -1.175841190977326e-15] # # altered_coef_table['r309'] = [4981.847585300046, 0.9953409249278389, 6.616819915490353e-09, # 7.072942793437885e-09, -4.7799815890757634e-12, 7.369734622022845e-16] # # altered_coef_table['r602'] = [4975.080088016758, 0.9916173886456268, 7.811003804278236e-06, # 1.1977785560589788e-09, -3.3762927213375386e-12, 7.593041888780153e-16] # # altered_coef_table['r715'] = [5014.023681360571, 0.99147302071155, 4.748885129798807e-06, # 3.1454713162197196e-09, -3.4683774647827705e-12, 6.101876288746191e-16] # handfit_fitting_dict = {} # handfit_fitting_dict['calib coefs'] = altered_coef_table # wm,fm = linelist['ThAr'] # else: # if pairnum==1 and self.camera=='r': # # altered_coef_table,thetype = self.filemanager.locate_calib_dict(fittype='full-ThAr', camera=self.camera, # # config=self.config, filenum=filenum) # # print(thetype, altered_coef_table) # altered_coef_table = self.filemanager.load_calib_dict(fittype='full-ThAr',cam=self.camera,config=self.config,filenum=1490,timestamp=679621) # initial_coef_table = Table(altered_coef_table['CALIB COEFS'].data) ## End HACK! handfit_fitting_dict, wm, fm = \ wavelength_fitting_by_line_selection(data, initial_coef_table,\ self.all_lines, linelist, self.mock_spec_w, self.mock_spec_f ,\ select_lines=select_lines,save_plots=self.save_plots,savetemplate_funcs=self.savetemplate_funcs,\ filenum=filenum,subset=hand_fit_subset,completed_coefs={}) if select_lines: linelistdict = {'ThAr': (wm, fm)} else: linelistdict = self.selected_lines if self.single_core: full_fitting_dict, badfits = \ auto_wavelength_fitting_by_lines(data, initial_coef_table, handfit_fitting_dict['calib coefs'].copy(), self.all_lines, linelistdict.copy(),\ mock_spec_w=mock_spec_w, mock_spec_f=mock_spec_f,\ filenum=filenum, \ save_plots=self.save_plots, savetemplate_funcs=self.savetemplate_funcs) # for datainfoname,datainfo in handfit_fitting_dict.items(): # for fib in datainfo.keys(): # full_fitting_dict[datainfoname][fib] = datainfo[fib] badfits = np.array(badfits) else: fib1s = self.instrument.lower_half_fibs[self.camera] fib2s = self.instrument.upper_half_fibs[self.camera] obs1 = { 'comp': data[fib1s.tolist()], 'fulllinelist': self.all_lines.copy(), 'coarse_coefs': initial_coef_table, 'linelistdict':linelistdict.copy(), \ 'mock_spec_w':mock_spec_w.copy(), 'mock_spec_f': mock_spec_f.copy(), \ 'out_coefs':handfit_fitting_dict['calib coefs'].copy(),'filenum':filenum, 'save_plots':self.save_plots, "savetemplate_funcs":self.savetemplate_funcs } obs2 = { 'comp': data[fib2s.tolist()], 'fulllinelist': self.all_lines.copy(), 'coarse_coefs': initial_coef_table.copy(), 'linelistdict':linelistdict.copy(), \ 'mock_spec_w':mock_spec_w.copy(), 'mock_spec_f': mock_spec_f.copy(), \ 'out_coefs':handfit_fitting_dict['calib coefs'].copy(),'filenum':filenum, 'save_plots': self.save_plots, "savetemplate_funcs": self.savetemplate_funcs } all_obs = [obs1, obs2] NPROC = np.clip(len(all_obs), 1, 4) with Pool(NPROC) as pool: tabs = pool.map(auto_wavelength_fitting_by_lines_wrapper, all_obs) full_fitting_dict, badfits = tabs[0] full_fitting_dict2, badfits2 = tabs[1] # ## The hand fit calibrations are in both returned dicts, remove from the second # ## Assign the other calibration info from hand fits to the output dicts # for datainfoname, datainfo in handfit_fitting_dict.items(): # ## use the autofitted wavelength solution even for hand fits, note we're not # ## assigning these values to the output array # if 'coef' in datainfoname: # for fib in datainfo.keys(): # full_fitting_dict2[datainfoname].pop(fib) # else: # for fib in datainfo.keys(): # full_fitting_dict[datainfoname][fib] = datainfo[fib] ## The hand fit calibrations are in both returned dicts, remove from the second ## Assign the other calibration info from hand fits to the output dicts for datainfoname, datainfo in full_fitting_dict2.items(): for fib in datainfo.keys(): full_fitting_dict[datainfoname][fib] = datainfo[fib] badfits = np.unique(np.append(badfits, badfits2)) handfit_bad_subset_dict, wm, fm = \ wavelength_fitting_by_line_selection(data, initial_coef_table, \ self.all_lines, linelistdict, self.mock_spec_w, self.mock_spec_f, \ select_lines=select_lines, save_plots=self.save_plots, savetemplate_funcs=self.savetemplate_funcs, \ filenum=filenum, subset=badfits, completed_coefs=full_fitting_dict['calib coefs'].copy()) for datainfoname, datainfo in handfit_bad_subset_dict.items(): for fib in datainfo.keys(): full_fitting_dict[datainfoname][fib] = datainfo[fib] if select_lines: self.selected_lines = full_fitting_dict['linelist'].copy() select_lines = False ## Zero pad rows so that the table won't throw an error for unequal sizes maxlams = int( np.max([ len(full_fitting_dict['wavelengths'][fib]) for fib in full_fitting_dict['wavelengths'].keys() ])) for fib in full_fitting_dict['wavelengths'].keys(): nlams = len(full_fitting_dict['wavelengths'][fib]) if nlams != maxlams: full_fitting_dict['wavelengths'][fib] = np.append( full_fitting_dict['wavelengths'][fib], np.zeros(maxlams - nlams)) full_fitting_dict['pixels'][fib] = np.append( full_fitting_dict['pixels'][fib], np.zeros(maxlams - nlams)) ## Create hdulist to export out_hdus = [ fits.PrimaryHDU(header=self.fine_calibrations[filenum].header) ] for out_name in output_names: curtab = Table() curdict = full_fitting_dict[out_name] for key in self.instrument.full_fibs[self.camera]: curtab.add_column(Table.Column(data=curdict[key], name=key)) out_hdus.append( fits.BinTableHDU(data=curtab.copy(), name=out_name)) hdulist = fits.HDUList(out_hdus) #out_calib_table = out_calib_table[np.sort(out_calib_table.colnames)] self.fine_calibration_coefs[pairnum] = full_fitting_dict[ 'calib coefs'].copy() if pairnum > 0: devs = find_devs(initial_coef_table, full_fitting_dict['calib coefs']) initial_coef_table = Table(full_fitting_dict['calib coefs'].copy()) self.final_calibrated_hdulists[pairnum] = hdulist self.filemanager.save_full_calib_dict(hdulist, self.lampstr_f, self.camera, self.config, filenum=filenum) gc.collect()
class TimeTable: masked = False def setup(self): # Initialize table self.table = Table(masked=self.masked) # Create column with mixed types np.random.seed(12345) self.table['i'] = np.arange(1000) self.table['a'] = np.random.random(1000) # float self.table['b'] = np.random.random(1000) > 0.5 # bool self.table['c'] = np.random.random((1000, 10)) # 2d column self.table['d'] = np.random.choice( np.array(list(string.ascii_letters)), 1000) self.np_table = np.array(self.table) self.extra_row = {'a': 1.2, 'b': True, 'c': np.repeat(1, 10), 'd': 'Z'} self.extra_column = np.random.randint(0, 100, 1000) self.row_indices = np.where(self.table['a'] > 0.9)[0] self.table_grouped = self.table.group_by('d') # Another table for testing joining self.other_table = Table(masked=self.masked) self.other_table['i'] = np.arange(1, 1000, 3) self.other_table['f'] = np.random.random() self.other_table.sort('f') # Another table for testing hstack self.other_table_2 = Table(masked=self.masked) self.other_table_2['g'] = np.random.random(1000) self.other_table_2['h'] = np.random.random((1000, 10)) self.bool_mask = self.table['a'] > 0.6 def time_table_slice_bool(self): table_subset = self.table[self.bool_mask] def time_table_slice_int(self): table_subset = self.table[self.row_indices] def time_column_slice_bool(self): col_subset = self.table['a'][self.bool_mask] def time_column_slice_int(self): col_subset = self.table['a'][self.row_indices] def time_column_get(self): self.table['c'] def time_column_make_bool_mask(self): self.table['a'] > 0.6 def time_multi_column_get(self): self.table[('a', 'c')] def time_column_set(self): self.table['a'] = 0. def time_column_set_all(self): self.table['b'][:] = True def time_column_set_row_subset(self): self.table['b'][self.bool_mask] = True def time_column_set_row_subset_int(self): self.table['b'][self.row_indices] = True def time_row_get(self): self.table[300] def time_iter_row(self): for row in self.table: pass def time_read_rows(self): for row in self.table: tuple(row) def time_item_get_rowfirst(self): self.table[300]['b'] def time_item_get_colfirst(self): self.table['b'][300] def time_add_row(self): self.table.add_row(self.extra_row) time_add_row.number = 1 time_add_row.repeat = 1 def time_remove_row(self): self.table.remove_row(6) time_remove_row.number = 1 time_remove_row.repeat = 1 def time_remove_rows(self): self.table.remove_rows(self.row_indices) time_remove_rows.number = 1 time_remove_rows.repeat = 1 def time_add_column(self): self.table['e'] = self.extra_column time_add_column.number = 1 time_add_column.repeat = 1 def time_remove_column(self): self.table.remove_column('a') time_remove_column.number = 1 time_remove_column.repeat = 1 def time_init_from_np_array_no_copy(self): Table(self.np_table, copy=False) def time_init_from_np_array_copy(self): Table(self.np_table, copy=True) def time_copy_table(self): self.table.copy() def time_copy_column(self): self.table['a'].copy() def time_group(self): self.table.group_by('d') def time_aggregate(self): # Test aggregate with a function that supports reduceat self.table_grouped.groups.aggregate(np.sum) def time_aggregate_noreduceat(self): # Test aggregate with a function that doesn't support reduceat self.table_grouped.groups.aggregate(lambda x: np.sum(x)) def time_sort(self): self.table.sort('a') def time_join_inner(self): join(self.table, self.other_table, keys="i", join_type='inner') def time_join_outer(self): join(self.table, self.other_table, keys="i", join_type='outer') def time_hstack(self): hstack([self.table, self.other_table_2]) def time_vstack(self): vstack([self.table, self.table])
def main(args): print '\n' for cat_path in args.catalogues: print 'reading %s..' % cat_path cat = fits.open(cat_path)[1].data t = Table(cat) # establish zmax per detection band for mcol, mlim, kcorr in zip(args.magcols, args.maglims, args.kcorrs): print '\t"%s" at limit: %s' % (mcol, mlim) print '\treading "%s" k-corrections..' % kcorr kcorr_list = pd.read_csv(kcorr, delimiter=' ') max_redshift = find_M_crossing(cat[mcol], mlim, kcorr_list) zmax_col = mcol+'_fl%.1f_zmax'%mlim t[zmax_col] = max_redshift t.write(cat_path, format='fits', overwrite=1) cat = t.copy() if args.randoms: print '\tbuilding clone randoms..' # drop points at random in 1x1 square Nrand = len(cat) * args.Nrand ra = np.random.rand(Nrand) dec = np.random.rand(Nrand) random_cols = [fitscol(array=ra, name='ra', format='D'), fitscol(array=dec, name='dec', format='D')] idcol = cat[args.idcol] zcol = np.asarray(cat[args.zcol]) mask = (zcol > 0) # apply any weighting if args.zweight is None: wcol = None bins = 'auto' else: wcol = cat[args.zweight] bins = np.linspace(zcol[mask].min(), zcol[mask].max(), 40) # fit to n(z) #print '\t\tfitting redshift distribution..' #zg, Pz = fit_smail(zcol[mask], quiet=1, weights=wcol, bins=bins).T zg = Pz = None # construct redshift distribution per band print '\t\tcloning..' for mcol, mlim in zip(args.magcols, args.maglims): zmax_col = mcol+'_fl%.1f_zmax'%mlim maxcol = cosmo.comoving_distance(cat[zmax_col]).value randoms_id_z = clone_galaxies(idcol, maxcol, args.Nrand, zg=zg, Pz=Pz, zlims=args.zlims) zmin, zmax = 0., randoms_id_z[:, 1].max() zgrid = np.linspace(zmin, zmax, 100) dgrid = cosmo.comoving_distance(zgrid) randoms_comoving = interp1d_(randoms_id_z[:, 1], zgrid, dgrid) random_cols.append(fitscol(array=randoms_id_z[:, 0], name=mcol+'_cloneID', format='K')) random_cols.append(fitscol(array=randoms_id_z[:, 1], name=mcol+'_cloneZ', format='D')) random_cols.append(fitscol(array=randoms_comoving, name=mcol+'_cloneComovingDist', format='D')) # save randoms catalogue with galaxy IDs and redshifts for selection random_cols = fits.ColDefs(random_cols) rand_hdu = fits.BinTableHDU.from_columns(random_cols) if args.o is None: out = cat_path.replace('.fits', '_CloneZIDRandoms.fits') else: out = args.o rand_hdu.writeto(out, overwrite=1) print 'done!'