def calculate_mags(sed_name, mag_norm, out_dict): """ Parameters ---------- sed_name is a numpy array of SED names mag_norm is a numpy array of magNorms out_dict is a multiprocessing.Manager.dict() that will store the magnitudes calculated by this process. """ i_process = mp.current_process().pid bp_dir = getPackageDir('throughputs') bp_dir = os.path.join(bp_dir, 'imsim', 'goal') bp_dict = BandpassDict.loadTotalBandpassesFromFiles(bandpassDir=bp_dir) sed_dir = getPackageDir('sims_sed_library') out_mags = np.zeros((len(sed_name), 6), dtype=float) for i_star, (s_name, m_norm) in enumerate(zip(sed_name, mag_norm)): spec = Sed() spec.readSED_flambda(os.path.join(sed_dir, defaultSpecMap[s_name])) fnorm = getImsimFluxNorm(spec, m_norm) spec.multiplyFluxNorm(fnorm) mags = bp_dict.magListForSed(spec) out_mags[i_star] = mags out_dict[i_process] = out_mags
def test_norm(self): """ Test that the special test case getImsimFluxNorm returns the same value as calling calcFluxNorm actually passing in the imsim Bandpass """ bp = Bandpass() bp.imsimBandpass() rng = np.random.RandomState(1123) wavelen = np.arange(300.0, 2000.0, 0.17) for ix in range(10): flux = rng.random_sample(len(wavelen))*100.0 sed = Sed() sed.setSED(wavelen=wavelen, flambda=flux) magmatch = rng.random_sample()*5.0 + 10.0 control = sed.calcFluxNorm(magmatch, bp) test = getImsimFluxNorm(sed, magmatch) # something about how interpolation is done in Sed means # that the values don't come out exactly equal. They come # out equal to 8 seignificant digits, though. self.assertEqual(control, test)
def _fluxes(sed_name, mag_norm, redshift): """ Find the fluxes for a galaxy component Parameters ---------- sed_name is an SED file name mag_norm is a float redshift is a float Returns ------- array of fluxes in ugrizy order """ if not hasattr(_fluxes, '_bp_dict'): bp_dir = getPackageDir('throughputs') bp_dir = os.path.join(bp_dir, 'imsim', 'goal') _fluxes._bp_dict = BandpassDict.loadTotalBandpassesFromFiles( bandpassDir=bp_dir) _fluxes._sed_dir = getPackageDir('sims_sed_library') spec = Sed() full_sed_name = os.path.join(_fluxes._sed_dir, sed_name) if not os.path.isfile(full_sed_name): full_sed_name = os.path.join(_fluxes._sed_dir, defaultSpecMap[sed_name]) spec.readSED_flambda(full_sed_name) fnorm = getImsimFluxNorm(spec, mag_norm) spec.multiplyFluxNorm(fnorm) spec.redshiftSED(redshift, dimming=True) return _fluxes._bp_dict.fluxListForSed(spec)
def test_norm(self): """ Test that the special test case getImsimFluxNorm returns the same value as calling calcFluxNorm actually passing in the imsim Bandpass """ bp = Bandpass() bp.imsimBandpass() rng = np.random.RandomState(1123) wavelen = np.arange(300.0, 2000.0, 0.17) for ix in range(10): flux = rng.random_sample(len(wavelen)) * 100.0 sed = Sed() sed.setSED(wavelen=wavelen, flambda=flux) magmatch = rng.random_sample() * 5.0 + 10.0 control = sed.calcFluxNorm(magmatch, bp) test = getImsimFluxNorm(sed, magmatch) # something about how interpolation is done in Sed means # that the values don't come out exactly equal. They come # out equal to 8 seignificant digits, though. self.assertEqual(control, test)
def _compute_SED(self): self._sed_obj = sims_photUtils.Sed() self._sed_obj.readSED_flambda(self.sed_file) fnorm = sims_photUtils.getImsimFluxNorm(self._sed_obj, self.mag_norm) self._sed_obj.multiplyFluxNorm(fnorm) if self.iAv != 0: self.shared_resources['ccm_model'].add_dust( self._sed_obj, self.iAv, self.iRv, 'intrinsic') if self.redshift != 0: self._sed_obj.redshiftSED(self.redshift, dimming=True) self._sed_obj.resampleSED(wavelen_match=self.bp_dict.wavelenMatch) if self.gAv != 0: self.shared_resources['ccm_model'].add_dust( self._sed_obj, self.gAv, self.gRv, 'Galactic')
def __init__(self, chunk): """ chunk = (simobjid, hpid, sedFilename, magNorm, ebv, varParamStr, parallax, ra, dec) """ # initialize some member variables expected by the # stellar variability models self.photParams = sims_photUtils.PhotometricParameters(nexp=1, exptime=30) self.lsstBandpassDict = sims_photUtils.BandpassDict.loadTotalBandpassesFromFiles( ) self._actually_calculated_columns = [] for bp in 'ugrizy': self._actually_calculated_columns.append('lsst_%s' % bp) sed_dir = os.environ['SIMS_SED_LIBRARY_DIR'] self.quiescent_mags = {} for bp in 'ugrizy': self.quiescent_mags[bp] = np.zeros(len(chunk), dtype=float) self.parallax = np.zeros(len(chunk), dtype=float) self.ebv = np.zeros(len(chunk), dtype=float) self.varParamStr = np.empty(len(chunk), dtype=(str, 300)) self.simobjid = np.empty(len(chunk), dtype=int) ccm_wav = None # find the quiescent magnitudes of the stars for i_star, star in enumerate(chunk): self.simobjid[i_star] = int(star[0]) full_file_name = os.path.join(sed_dir, defaultSpecMap[star[2]]) spec = sims_photUtils.Sed() spec.readSED_flambda(full_file_name) fnorm = sims_photUtils.getImsimFluxNorm(spec, float(star[3])) spec.multiplyFluxNorm(fnorm) if ccm_wav is None or not np.array_equal(spec.wavelen, ccm_wav): ccm_wav = np.copy(spec.wavelen) a_x, b_x = spec.setupCCM_ab() ebv_val = float(star[4]) spec.addDust(a_x, b_x, ebv=ebv_val, R_v=3.1) mag_list = self.lsstBandpassDict.magListForSed(spec) for i_bp, bp in enumerate('ugrizy'): self.quiescent_mags[bp][i_star] = mag_list[i_bp] self.ebv[i_star] = ebv_val self.varParamStr[i_star] = star[5] self.parallax[i_star] = float(star[6]) self.parallax *= (np.pi / 648000000.0) # convert from mas to radians
def _create_sed(self): """ Function to create the lsst.sims.photUtils Sed object. """ self.sed = sims_photUtils.Sed() self.sed.readSED_flambda(self.sed_file) fnorm = sims_photUtils.getImsimFluxNorm(self.sed, self.mag_norm) self.sed.multiplyFluxNorm(fnorm) if self.iAv != 0: self.add_dust(self.iAv, self.iRv, 'intrinsic') if self.redshift != 0: self.sed.redshiftSED(self.redshift, dimming=True) self.sed.resampleSED(wavelen_match=self.bp_dict.wavelenMatch) if self.gAv != 0: self.add_dust(self.gAv, self.gRv, 'Galactic')
def process_component(sed_names, ebv_arr, redshift, sed_dexes, av, rv, magnorm, my_lock, output_dict, galaxy_id): (dummy, hw_bp_dict) = sims_photUtils.BandpassDict.loadBandpassesFromFiles() n_obj = len(sed_dexes) fluxes = np.zeros((6, n_obj), dtype=float) fluxes_noMW = np.zeros((6, n_obj), dtype=float) t_start = time.time() for sed_id in np.unique(sed_dexes): valid = np.where(sed_dexes == sed_id) sed_full_name = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], sed_names[sed_id]) spec_rest = sims_photUtils.Sed() spec_rest.readSED_flambda(sed_full_name, cache_sed=False) a_x_rest, b_x_rest = spec_rest.setupCCM_ab() fnorm_at_21 = sims_photUtils.getImsimFluxNorm(spec_rest, 21.0) for g_dex in valid[0]: spec_rest_dust = sims_photUtils.Sed(wavelen=spec_rest.wavelen, flambda=spec_rest.flambda) spec_rest_dust.addDust(a_x_rest, b_x_rest, A_v=av[g_dex], R_v=rv[g_dex]) spec_rest_dust.redshiftSED(redshift[g_dex], dimming=True) flux_list_noMW = hw_bp_dict.fluxListForSed(spec_rest_dust) a_x, b_x = spec_rest_dust.setupCCM_ab() spec_rest_dust.addDust(a_x, b_x, R_v=3.1, ebv=ebv_arr[g_dex]) flux_list = hw_bp_dict.fluxListForSed(spec_rest_dust) for i_bp, bp in enumerate('ugrizy'): factor = fnorm_at_21 * np.power( 10.0, -0.4 * (magnorm[i_bp][g_dex] - 21.0)) fluxes[i_bp][g_dex] = factor * flux_list[i_bp] fluxes_noMW[i_bp][g_dex] = factor * flux_list_noMW[i_bp] with my_lock as context: print('adding %d to fluxes' % len(fluxes)) output_dict['fluxes'].append(fluxes) output_dict['fluxes_noMW'].append(fluxes_noMW) output_dict['galaxy_id'].append(galaxy_id)
def add_flux(self, sed_name, redshift, magnorm_dict, av, rv, bp_dict=None): if bp_dict is None: bp_dict = BandpassDict.loadTotalBandpassesFromFiles() result_dict_no_mw = {} result_dict_mw = {} sed_obj = Sed() sed_obj.readSED_flambda(os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], sed_name)) a_x, b_x = sed_obj.setupCCM_ab() for bandpass_name in bp_dict.keys(): sed_copy = deepcopy(sed_obj) flux_norm = getImsimFluxNorm(sed_copy, magnorm_dict[bandpass_name]) sed_copy.multiplyFluxNorm(flux_norm) sed_copy.redshiftSED(redshift, dimming=True) band_flux = sed_copy.calcFlux(bp_dict[bandpass_name]) result_dict_no_mw[bandpass_name] = band_flux sed_copy.addDust(a_x, b_x, A_v=av, R_v=rv) band_flux_mw = sed_copy.calcFlux(bp_dict[bandpass_name]) result_dict_mw[bandpass_name] = band_flux_mw return result_dict_no_mw, result_dict_mw
query += 'galaxy WHERE varParamStr IS NOT NULL ' query += 'AND dec BETWEEN -2.5 AND 2.5 ' query += 'AND (ra<2.5 OR ra>357.5)' dtype = np.dtype([('magnorm', float), ('redshift', float), ('varParamStr', str, 400)]) data_iter = db.get_arbitrary_chunk_iterator(query, dtype=dtype, chunk_size=10000) with open('data/dc1_agn_params.txt', 'w') as out_file: out_file.write('# z m_i M_i tau sfu sfg sfr sfi sfz sfy\n') for chunk in data_iter: DM = cosmo.distanceModulus(redshift=chunk['redshift']) k_corr = np.interp(chunk['redshift'], z_grid, k_grid) for i_row, agn in enumerate(chunk): ss = Sed(wavelen=base_sed.wavelen, flambda=base_sed.flambda) fnorm = getImsimFluxNorm(ss, agn['magnorm']) ss.multiplyFluxNorm(fnorm) ss.redshiftSED(agn['redshift'], dimming=True) mag = ss.calcMag(bp) abs_m_i = mag - DM[i_row] - k_corr[i_row] params = json.loads(agn['varParamStr'])['pars'] out_file.write( '%e %e %e %e %e %e %e %e %e %e\n' % (agn['redshift'], mag, abs_m_i, params['agn_tau'], params['agn_sfu'], params['agn_sfg'], params['agn_sfr'], params['agn_sfi'], params['agn_sfz'], params['agn_sfy']))
def loadSedsFromList(self, sedNameList, magNormList, \ internalAvList=None, galacticAvList=None, redshiftList=None): """ Load the Seds specified by sedNameList, applying the specified normalization, extinction, and redshift. @param [in] sedList is a list of file names containing Seds @param [in] magNorm is the magnitude normalization @param [in] internalAvList is an optional list of A(V) due to internal dust @param [in] galacticAvList is an optional list of A(V) due to Milky Way dust @param [in] redshiftList is an optional list of redshifts for the input Sed Seds are read in and stored to this object's internal list of Seds. Note: if you constructed this SedList object without internalAvList, you cannot load Seds with internalAvList now. Likewise for galacticAvlist and redshiftList. """ if not self._initialized: if internalAvList is not None: self._internal_av_list = copy.deepcopy(list(internalAvList)) else: self._internal_av_list = None if galacticAvList is not None: self._galactic_av_list = copy.deepcopy(list(galacticAvList)) else: self._galactic_av_list = None if redshiftList is not None: self._redshift_list = copy.deepcopy(list(redshiftList)) else: self._redshift_list = None else: if self._internal_av_list is None and internalAvList is not None: raise RuntimeError( "This SedList does not contain internalAvList") elif self._internal_av_list is not None: if internalAvList is None: self._internal_av_list += [None] * len(sedNameList) else: self._internal_av_list += list(internalAvList) if self._galactic_av_list is None and galacticAvList is not None: raise RuntimeError( "This SedList does not contain galacticAvList") elif self._galactic_av_list is not None: if galacticAvList is None: self._galactic_av_list += [None] * len(sedNameList) else: self._galactic_av_list += list(galacticAvList) if self._redshift_list is None and redshiftList is not None: raise RuntimeError( "This SedList does not contain redshiftList") elif self._redshift_list is not None: if redshiftList is None: self._redshift_list += [None] * len(sedNameList) else: self._redshift_list += list(redshiftList) temp_sed_list = [] for sedName, magNorm in zip(sedNameList, magNormList): sed = Sed() if sedName != "None": if self._spec_map is not None: sed.readSED_flambda( os.path.join(self._file_dir, self._spec_map[sedName])) else: sed.readSED_flambda(os.path.join(self._file_dir, sedName)) if self._normalizing_bandpass is not None: fNorm = sed.calcFluxNorm(magNorm, self._normalizing_bandpass) else: fNorm = getImsimFluxNorm(sed, magNorm) sed.multiplyFluxNorm(fNorm) temp_sed_list.append(sed) if internalAvList is not None: self._av_int_wavelen, \ self._a_int, \ self._b_int = self.applyAv(temp_sed_list, internalAvList, self._av_int_wavelen, self._a_int, self._b_int) if redshiftList is not None: self.applyRedshift(temp_sed_list, redshiftList) if self._wavelen_match is not None: for sedObj in temp_sed_list: if sedObj.wavelen is not None: sedObj.resampleSED(wavelen_match=self._wavelen_match) if galacticAvList is not None: self._av_gal_wavelen, \ self._a_gal, \ self._b_gal = self.applyAv(temp_sed_list, galacticAvList, self._av_gal_wavelen, self._a_gal, self._b_gal) self._sed_list += temp_sed_list self._initialized = True
size=n_to_validate) else: dexes_to_validate = np.arange(len(control_qties['redshift']), dtype=int) max_offset = -1.0 sed_dir = os.environ['SIMS_SED_LIBRARY_DIR'] tot_bp_dict = BandpassDict.loadTotalBandpassesFromFiles() for ii in dexes_to_validate: disk_name = os.path.join(sed_dir, sed_names[disk_sed_idx[ii]].decode()) bulge_name = os.path.join(sed_dir, sed_names[bulge_sed_idx[ii]].decode()) for i_bp, bp in enumerate('ugrizy'): disk_s = Sed() disk_s.readSED_flambda(disk_name) fnorm = getImsimFluxNorm(disk_s, disk_magnorm[i_bp][ii]) disk_s.multiplyFluxNorm(fnorm) ax, bx = disk_s.setupCCM_ab() disk_s.addDust(ax, bx, A_v=disk_av[ii], R_v=disk_rv[ii]) disk_s.redshiftSED(control_qties['redshift'][ii], dimming=True) disk_f = disk_s.calcFlux(tot_bp_dict[bp]) bulge_s = Sed() bulge_s.readSED_flambda(bulge_name) fnorm = getImsimFluxNorm(bulge_s, bulge_magnorm[i_bp][ii]) bulge_s.multiplyFluxNorm(fnorm) ax, bx = bulge_s.setupCCM_ab() bulge_s.addDust(ax, bx, A_v=bulge_av[ii], R_v=bulge_rv[ii]) bulge_s.redshiftSED(control_qties['redshift'][ii], dimming=True) bulge_f = bulge_s.calcFlux(tot_bp_dict[bp])
sed_arr = sed_fit_file['%s_sed' % component][()][subset] av_arr = sed_fit_file['%s_av' % component][()][subset] rv_arr = sed_fit_file['%s_rv' % component][()][subset] mn_arr = sed_fit_file['%s_magnorm' % component][()][i_bp, :][subset] z_arr = cosmoDC2_data['redshift'][crossmatch_dex] for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate( zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)): # read in the SED file from the library sed_file_name = os.path.join(sed_lib_dir, sed_names[s_dex]) sed = sims_photUtils.Sed() sed.readSED_flambda(sed_file_name) # find and apply normalizing flux fnorm = sims_photUtils.getImsimFluxNorm(sed, mn) sed.multiplyFluxNorm(fnorm) # add internal dust if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w): ccm_w = np.copy(sed.wavelen) a_x, b_x = sed.setupCCM_ab() sed.addDust(a_x, b_x, A_v=av, R_v=rv) # apply redshift sed.redshiftSED(zz, dimming=True) # flux, in Janskys, without Milky Way dust extinction f_noMW = sed.calcFlux(lsst_bp_dict[bp]) # apply Milky Way dust
def process_instance_catalog(catalog_name, centroid_dir, bp_dict): sed_dir = getPackageDir('sims_sed_library') i_filter = -1 obshistid = -1 with open(catalog_name, 'r') as in_file: for line in in_file: params = line.strip().split() if params[0] == 'filter': i_filter = int(params[1]) if params[0] == 'obshistid': obshistid = int(params[1]) if i_filter >= 0 and obshistid >= 0: break filter_name = 'ugrizy'[i_filter] bp = bp_dict[filter_name] objid_arr = [] flux_arr = [] redshift_arr = [] magnorm_arr = [] phot_params = PhotometricParameters(nexp=1, exptime=30.0, gain=1.0) with open(catalog_name, 'r') as in_file: for line in in_file: params = line.strip().split() if params[0] != 'object': continue sed_name = params[5] obj_id = int(params[1]) magnorm = float(params[4]) redshift = float(params[6]) spec = Sed() spec.readSED_flambda(os.path.join(sed_dir, sed_name)) fnorm = getImsimFluxNorm(spec, magnorm) spec.multiplyFluxNorm(fnorm) spec.redshiftSED(redshift, dimming=True) adu = spec.calcADU(bp, photParams=phot_params) objid_arr.append(obj_id) flux_arr.append(adu) redshift_arr.append(redshift) magnorm_arr.append(magnorm) centroid_files = os.listdir(centroid_dir) phosim_objid_arr = [] phosim_flux_arr = [] x_arr = [] y_arr = [] chip_name_arr = [] dtype = np.dtype([('id', int), ('phot', float), ('x', float), ('y', float)]) for file_name in centroid_files: if 'e_%d' % obshistid not in file_name or 'f%d' % i_filter not in file_name: continue name_params = file_name.split('_') raft = name_params[5] sensor = name_params[6] chip_name = '%s:%s,%s %s:%s,%s' % (raft[0], raft[1], raft[2], sensor[0], sensor[1], sensor[2]) full_name = os.path.join(centroid_dir, file_name) data = np.genfromtxt(full_name, dtype=dtype, skip_header=1) for line in data: phosim_objid_arr.append(line['id']) phosim_flux_arr.append(line['phot']) x_arr.append(line['x']) y_arr.append(line['y']) chip_name_arr.append(chip_name) objid_arr = np.array(objid_arr) flux_arr = np.array(flux_arr) redshift_arr = np.array(redshift_arr) magnorm_arr = np.array(magnorm_arr) phosim_objid_arr = np.array(phosim_objid_arr) phosim_flux_arr = np.array(phosim_flux_arr) x_arr = np.array(x_arr) y_arr = np.array(y_arr) chip_name_arr = np.array(chip_name_arr) sorted_dex = np.argsort(objid_arr) objid_arr = objid_arr[sorted_dex] flux_arr = flux_arr[sorted_dex] redshift_arr = redshift_arr[sorted_dex] magnorm_arr = magnorm_arr[sorted_dex] sorted_dex = np.argsort(phosim_objid_arr) phosim_objid_arr = phosim_objid_arr[sorted_dex] phosim_flux_arr = phosim_flux_arr[sorted_dex] x_arr = x_arr[sorted_dex] y_arr = y_arr[sorted_dex] chip_name_arr = chip_name_arr[sorted_dex] np.testing.assert_array_equal(phosim_objid_arr, objid_arr) return (objid_arr, flux_arr, phosim_flux_arr, redshift_arr, magnorm_arr, x_arr, y_arr, chip_name_arr, filter_name)
def load_dc2_hosts(catalog_version, agn_db, sed_dir): # Load DC2 lens and host catalogs dc2_reader = DC2Reader(catalog_version) host_cat_filters = [ 'ra < 75.', 'ra > 72.5', 'dec < -42.5', 'dec > -45.', 'redshift_true > 0.25' ] host_galaxy_df = dc2_reader.load_galaxy_catalog(host_cat_filters) # Load DC2 AGN host_agn_query = 'ra > 72.5 and ra < 75. and dec > -45 and dec < -42.5 and redshift > 0.25' host_agn_df = dc2_reader.load_agn_catalog(agn_db, host_agn_query) # Load healpix ids so we can get SED information healpix_ids_hosts = get_healpix_id(host_galaxy_df['ra'], host_galaxy_df['dec']) host_galaxy_df['healpix_id'] = healpix_ids_hosts # To make SED loading quicker (we have to load an entire healpixel worth of SED # info at a time) we will only take the four healpixels with the most galaxies. # For more info see notebook matchingLensGalaxies.ipynb healpix_query = 'healpix_id < 10203 or healpix_id == 10329 or healpix_id == 10452' host_galaxy_df = host_galaxy_df.query(healpix_query).reset_index(drop=True) # Join host and agn info host_full_df = host_galaxy_df.join(host_agn_df.set_index('galaxy_id'), on='galaxy_id', rsuffix='_agn') host_full_df = host_full_df.rename( columns={ 'magNorm': 'magNorm_agn', 'varParamStr': 'varParamStr_agn', 'redshift': 'redshift_agn' }) # Pick out the host galaxies that have an AGN # These will be our potential matches for lensed AGN agn_host_full_df = host_full_df.query('ra_agn > -99.').reset_index( drop=True) # Host galaxies without an AGN are our pool of potential SNe host galaxies sne_host_full_df = host_full_df.iloc[np.where( np.isnan(host_full_df['M_i'].values))] # Load i-band mags for AGN so we can match to OM10 lsst_bp_dict = BandpassDict.loadTotalBandpassesFromFiles() agn_sed = Sed() agn_sed.readSED_flambda( os.path.join(getPackageDir('SIMS_SED_LIBRARY'), 'agnSED', 'agn.spec.gz')) mag_i_agn = [] for agn_data in agn_host_full_df[['magNorm_agn', 'redshift_true']].values: agn_mag, agn_redshift = agn_data if np.isnan(agn_mag): mag_i_agn.append(np.nan) continue agn_copy = deepcopy(agn_sed) flux_norm = getImsimFluxNorm(agn_copy, agn_mag) agn_copy.multiplyFluxNorm(flux_norm) agn_copy.redshiftSED(agn_redshift, dimming=True) mag_i_agn.append(agn_copy.calcMag(lsst_bp_dict['i'])) agn_host_full_df['mag_i_agn'] = mag_i_agn sed_df = pd.DataFrame([], columns=[ 'bulge_av', 'bulge_rv', 'bulge_sed', 'disk_av', 'disk_rv', 'disk_sed', 'galaxy_id', ]) for hpix in np.unique(host_full_df['healpix_id']): # This file does not exist. And there are only 19 galaxies in this healpix anyway. if hpix == 10572: continue f = h5py.File(os.path.join(sed_dir, 'sed_fit_%i.h5' % hpix)) sed_hpix_df = pd.DataFrame([]) band_list = ['u', 'g', 'r', 'i', 'z', 'y'] sed_names = f['sed_names'][()] for key in list(f.keys()): print(key, f[key].len()) # if (key.endswith('fluxes') or key.endswith('magnorm')): if key.endswith('magnorm'): key_data = f[key][()] for i in range(6): sed_hpix_df[str(key + '_' + band_list[i])] = key_data[i] elif key in ['sed_names', 'ra', 'dec']: continue elif key.endswith('fluxes'): continue elif key.endswith('sed'): sed_hpix_df[key] = [sed_names[idx] for idx in f[key][()]] else: sed_hpix_df[key] = f[key][()] sed_df = pd.concat([sed_df, sed_hpix_df], sort=False) sed_df = sed_df.reset_index(drop=True) sed_df = sed_df.drop(columns=['redshift']) agn_host_join = agn_host_full_df.join(sed_df.set_index('galaxy_id'), on='galaxy_id') sne_host_join = sne_host_full_df.join(sed_df.set_index('galaxy_id'), on='galaxy_id') # Clean up galaxies with bad magnorms good_agn_rows = np.where((~np.isnan(agn_host_join['bulge_magnorm_u']) & ~np.isinf(agn_host_join['bulge_magnorm_u']) & ~np.isnan(agn_host_join['disk_magnorm_u']) & ~np.isinf(agn_host_join['disk_magnorm_u'])))[0] agn_host_final = agn_host_join.iloc[good_agn_rows].reset_index(drop=True) good_sne_rows = np.where((~np.isnan(sne_host_join['bulge_magnorm_u']) & ~np.isinf(sne_host_join['bulge_magnorm_u']) & ~np.isnan(sne_host_join['disk_magnorm_u']) & ~np.isinf(sne_host_join['disk_magnorm_u'])))[0] sne_host_final = sne_host_join.iloc[good_sne_rows].reset_index(drop=True) return agn_host_final, sne_host_final
idlrest = i_rest_dustless[i_star] zdlrest = z_rest_dustless[i_star] ydlrest = y_rest_dustless[i_star] dustless = Sed() sed = Sed() rest_sed = Sed() rest_sed_dustless = Sed() full_sed_name = os.path.join(gal_sed_dir, sed_name) sed.readSED_flambda(full_sed_name) dustless.readSED_flambda(full_sed_name) rest_sed.readSED_flambda(full_sed_name) rest_sed_dustless.readSED_flambda(full_sed_name) f_norm = getImsimFluxNorm(sed, mag_norm) sed.multiplyFluxNorm(f_norm) dustless.multiplyFluxNorm(f_norm) f_norm = getImsimFluxNorm(rest_sed, rest_mag_norm) rest_sed.multiplyFluxNorm(f_norm) rest_sed_dustless.multiplyFluxNorm(f_norm) a_x, b_x = sed.setupCCMab() R_v = av/ebv sed.addCCMDust(a_x, b_x, ebv=ebv, R_v=R_v) rest_sed.addCCMDust(a_x, b_x, ebv=ebv, R_v=R_v) sed.redshiftSED(full_redshift, dimming=True) dustless.redshiftSED(full_redshift, dimming=True)
def loadSedsFromList(self, sedNameList, magNormList, \ internalAvList=None, galacticAvList=None, redshiftList=None): """ Load the Seds specified by sedNameList, applying the specified normalization, extinction, and redshift. @param [in] sedList is a list of file names containing Seds @param [in] magNorm is the magnitude normalization @param [in] internalAvList is an optional list of A(V) due to internal dust @param [in] galacticAvList is an optional list of A(V) due to Milky Way dust @param [in] redshiftList is an optional list of redshifts for the input Sed Seds are read in and stored to this object's internal list of Seds. Note: if you constructed this SedList object without internalAvList, you cannot load Seds with internalAvList now. Likewise for galacticAvlist and redshiftList. """ if not self._initialized: if internalAvList is not None: self._internal_av_list = copy.deepcopy(list(internalAvList)) else: self._internal_av_list = None if galacticAvList is not None: self._galactic_av_list = copy.deepcopy(list(galacticAvList)) else: self._galactic_av_list = None if redshiftList is not None: self._redshift_list = copy.deepcopy(list(redshiftList)) else: self._redshift_list = None else: if self._internal_av_list is None and internalAvList is not None: raise RuntimeError("This SedList does not contain internalAvList") elif self._internal_av_list is not None: if internalAvList is None: self._internal_av_list += [None] * len(sedNameList) else: self._internal_av_list += list(internalAvList) if self._galactic_av_list is None and galacticAvList is not None: raise RuntimeError("This SedList does not contain galacticAvList") elif self._galactic_av_list is not None: if galacticAvList is None: self._galactic_av_list += [None] * len(sedNameList) else: self._galactic_av_list += list(galacticAvList) if self._redshift_list is None and redshiftList is not None: raise RuntimeError("This SedList does not contain redshiftList") elif self._redshift_list is not None: if redshiftList is None: self._redshift_list += [None] * len(sedNameList) else: self._redshift_list += list(redshiftList) temp_sed_list = [] for sedName, magNorm in zip(sedNameList, magNormList): sed = Sed() if sedName != "None": if self._spec_map is not None: sed.readSED_flambda(os.path.join(self._file_dir, self._spec_map[sedName])) else: sed.readSED_flambda(os.path.join(self._file_dir, sedName)) if self._normalizing_bandpass is not None: fNorm = sed.calcFluxNorm(magNorm, self._normalizing_bandpass) else: fNorm = getImsimFluxNorm(sed, magNorm) sed.multiplyFluxNorm(fNorm) temp_sed_list.append(sed) if internalAvList is not None: self._av_int_wavelen, \ self._a_int, \ self._b_int = self.applyAv(temp_sed_list, internalAvList, self._av_int_wavelen, self._a_int, self._b_int) if redshiftList is not None: self.applyRedshift(temp_sed_list, redshiftList) if self._wavelen_match is not None: for sedObj in temp_sed_list: if sedObj.wavelen is not None: sedObj.resampleSED(wavelen_match=self._wavelen_match) if galacticAvList is not None: self._av_gal_wavelen, \ self._a_gal, \ self._b_gal = self.applyAv(temp_sed_list, galacticAvList, self._av_gal_wavelen, self._a_gal, self._b_gal) self._sed_list += temp_sed_list self._initialized = True
t_id = twinkles_data['twinklesId'][idx] sed_name = twinkles_data['lens_sed'][idx] i_magnorm = twinkles_data['sed_magNorm'][idx][3] apmag_i = twinkles_data['APMAG_I'][idx] av = twinkles_data['lens_av'][idx] rv = twinkles_data['lens_rv'][idx] #valid = np.where(gal_to_twink['twinkles_id']==t_id)[0] #g_id = gal_to_twink['galaxy_id'][valid] #valid = np.where(galaxy_id==g_id)[0] zz = twinkles_data['ZLENS'][idx] spec = sims_photUtils.Sed() full_file_name = os.path.join(sed_dir, sed_name) spec.readSED_flambda(full_file_name) fnorm = sims_photUtils.getImsimFluxNorm(spec, i_magnorm) spec.multiplyFluxNorm(fnorm) if dust_wav is None or not np.array_equal(spec.wavelen, dust_wav): dust_wav = np.copy(spec.wavelen) a_x, b_x = spec.setupCCM_ab() spec.addDust(a_x, b_x, A_v=av, R_v=rv) spec.redshiftSED(zz, dimming=True) i_mag = spec.calcMag(bp_dict['i']) d_mag = i_mag-apmag_i abs_d_mag = np.abs(i_mag-apmag_i) if abs_d_mag > max_dmag: print('d_mag %e: is %e shld %e; %d' % (d_mag, i_mag, apmag_i, twinkles_data['LENSID'][idx])) print('av %e rv %e zz %e sed %s magnorm %e AP %e' % (av, rv, zz, sed_name,i_magnorm,apmag_i)) max_dmag = abs_d_mag
def validate_chunk(data_in, in_dir, healpix, read_lock, write_lock, output_dict): galaxy_id = data_in['galaxy_id'] redshift = data_in['redshift'] mag_in = {} for bp in 'ugrizy': mag_in[bp] = data_in['mag_true_%s_lsst' % bp] eps = 1.0e-20 disk_to_bulge = {} for bp in 'ugrizy': disk_name = 'LSST_filters/diskLuminositiesStellar:' disk_name += 'LSST_%s:observed:dustAtlas' % bp bulge_name = 'LSST_filters/spheroidLuminositiesStellar:' bulge_name += 'LSST_%s:observed:dustAtlas' % bp denom = np.where(data_in[bulge_name]>0.0, data_in[bulge_name], eps) disk_to_bulge[bp] = data_in[disk_name]/denom assert np.isfinite(disk_to_bulge[bp]).sum()==len(disk_to_bulge[bp]) sed_dir = os.environ['SIMS_SED_LIBRARY_DIR'] tot_bp_dict = photUtils.BandpassDict.loadTotalBandpassesFromFiles() fit_file = os.path.join(in_dir, 'sed_fit_%d.h5' % healpix) assert os.path.isfile(fit_file) with read_lock: with h5py.File(fit_file, 'r') as in_file: sed_names = in_file['sed_names'][()] galaxy_id_fit = in_file['galaxy_id'][()] valid = np.in1d(galaxy_id_fit, galaxy_id) galaxy_id_fit = galaxy_id_fit[valid] np.testing.assert_array_equal(galaxy_id_fit, galaxy_id) disk_magnorm = in_file['disk_magnorm'][()][:,valid] disk_av = in_file['disk_av'][()][valid] disk_rv = in_file['disk_rv'][()][valid] disk_sed = in_file['disk_sed'][()][valid] bulge_magnorm = in_file['bulge_magnorm'][()][:,valid] bulge_av = in_file['bulge_av'][()][valid] bulge_rv = in_file['bulge_rv'][()][valid] bulge_sed = in_file['bulge_sed'][()][valid] sed_names = [name.decode() for name in sed_names] local_worst = {} local_worst_ratio = {} ccm_w = None dummy_sed = photUtils.Sed() for i_obj in range(len(disk_av)): for i_bp, bp in enumerate('ugrizy'): d_sed = photUtils.Sed() fname = os.path.join(sed_dir, sed_names[disk_sed[i_obj]]) d_sed.readSED_flambda(fname) fnorm = photUtils.getImsimFluxNorm(d_sed, disk_magnorm[i_bp][i_obj]) d_sed.multiplyFluxNorm(fnorm) if ccm_w is None or not np.array_equal(d_sed.wavelen, ccm_w): ccm_w = np.copy(d_sed.wavelen) ax, bx = d_sed.setupCCM_ab() d_sed.addDust(ax, bx, R_v=disk_rv[i_obj], A_v=disk_av[i_obj]) d_sed.redshiftSED(redshift[i_obj], dimming=True) d_flux = d_sed.calcFlux(tot_bp_dict[bp]) b_sed = photUtils.Sed() fname = os.path.join(sed_dir, sed_names[bulge_sed[i_obj]]) b_sed.readSED_flambda(fname) fnorm = photUtils.getImsimFluxNorm(b_sed, bulge_magnorm[i_bp][i_obj]) b_sed.multiplyFluxNorm(fnorm) if ccm_w is None or not np.array_equal(b_sed.wavelen, ccm_w): ccm_w = np.copy(b_sed.wavelen) ax, bx = b_sed.setupCCM_ab() b_sed.addDust(ax, bx, R_v=bulge_rv[i_obj], A_v=bulge_av[i_obj]) b_sed.redshiftSED(redshift[i_obj], dimming=True) b_flux = b_sed.calcFlux(tot_bp_dict[bp]) flux_ratio = d_flux/max(b_flux, eps) if flux_ratio<1.0e3 or disk_to_bulge[bp][i_obj]<1.0e3: delta_ratio = np.abs(flux_ratio-disk_to_bulge[bp][i_obj]) if disk_to_bulge[bp][i_obj]>eps: delta_ratio /= disk_to_bulge[bp][i_obj] if bp not in local_worst_ratio or delta_ratio>local_worst_ratio[bp]: local_worst_ratio[bp] = delta_ratio tot_flux = b_flux + d_flux true_flux = dummy_sed.fluxFromMag(mag_in[bp][i_obj]) delta_flux = np.abs(1.0-tot_flux/true_flux) if bp not in local_worst or delta_flux>local_worst[bp]: local_worst[bp] = delta_flux with write_lock: for bp in 'ugrizy': if bp not in output_dict or local_worst[bp]>output_dict[bp]: output_dict[bp] = local_worst[bp] ratio_name = '%s_ratio' % bp if (ratio_name not in output_dict or local_worst_ratio[bp]>output_dict[ratio_name]): output_dict[ratio_name] = local_worst_ratio[bp]
def sources_from_file(file_name, obs_md, phot_params, numRows=None): """ Read in an InstanceCatalog and extract all of the astrophysical sources from it Parameters ---------- file_name: str The name of the InstanceCatalog obs_md: ObservationMetaData The ObservationMetaData characterizing the pointing phot_params: PhotometricParameters The PhotometricParameters characterizing this telescope numRows: int (optional) The number of rows of the InstanceCatalog to read in (including the header) Returns ------- gs_obj_arr: numpy array Contains the GalSimCelestialObjects for all of the astrophysical sources in this InstanceCatalog out_obj_dict: dict Keyed on the names of the detectors in the LSST camera. The values are numpy arrays of GalSimCelestialObjects that should be simulated for that detector, including objects that are near the edge of the chip or just bright (in which case, they might still illuminate the detector). """ camera = get_obs_lsstSim_camera() num_objects = 0 ct_rows = 0 with fopen(file_name, mode='rt') as input_file: for line in input_file: ct_rows += 1 params = line.strip().split() if params[0] == 'object': num_objects += 1 if numRows is not None and ct_rows >= numRows: break # RA, Dec in the coordinate system expected by PhoSim ra_phosim = np.zeros(num_objects, dtype=float) dec_phosim = np.zeros(num_objects, dtype=float) sed_name = [None] * num_objects mag_norm = 55.0 * np.ones(num_objects, dtype=float) gamma1 = np.zeros(num_objects, dtype=float) gamma2 = np.zeros(num_objects, dtype=float) kappa = np.zeros(num_objects, dtype=float) internal_av = np.zeros(num_objects, dtype=float) internal_rv = np.zeros(num_objects, dtype=float) galactic_av = np.zeros(num_objects, dtype=float) galactic_rv = np.zeros(num_objects, dtype=float) semi_major_arcsec = np.zeros(num_objects, dtype=float) semi_minor_arcsec = np.zeros(num_objects, dtype=float) position_angle_degrees = np.zeros(num_objects, dtype=float) sersic_index = np.zeros(num_objects, dtype=float) npoints = np.zeros(num_objects, dtype=int) redshift = np.zeros(num_objects, dtype=float) unique_id = np.zeros(num_objects, dtype=int) object_type = np.zeros(num_objects, dtype=int) i_obj = -1 with fopen(file_name, mode='rt') as input_file: for line in input_file: params = line.strip().split() if params[0] != 'object': continue i_obj += 1 if numRows is not None and i_obj >= num_objects: break unique_id[i_obj] = int(params[1]) ra_phosim[i_obj] = float(params[2]) dec_phosim[i_obj] = float(params[3]) mag_norm[i_obj] = float(params[4]) sed_name[i_obj] = params[5] redshift[i_obj] = float(params[6]) gamma1[i_obj] = float(params[7]) gamma2[i_obj] = float(params[8]) kappa[i_obj] = float(params[9]) if params[12].lower() == 'point': object_type[i_obj] = _POINT_SOURCE i_gal_dust_model = 14 if params[13].lower() != 'none': i_gal_dust_model = 16 internal_av[i_obj] = float(params[14]) internal_rv[i_obj] = float(params[15]) if params[i_gal_dust_model].lower() != 'none': galactic_av[i_obj] = float(params[i_gal_dust_model + 1]) galactic_rv[i_obj] = float(params[i_gal_dust_model + 2]) elif params[12].lower() == 'sersic2d': object_type[i_obj] = _SERSIC_2D semi_major_arcsec[i_obj] = float(params[13]) semi_minor_arcsec[i_obj] = float(params[14]) position_angle_degrees[i_obj] = float(params[15]) sersic_index[i_obj] = float(params[16]) i_gal_dust_model = 18 if params[17].lower() != 'none': i_gal_dust_model = 20 internal_av[i_obj] = float(params[18]) internal_rv[i_obj] = float(params[19]) if params[i_gal_dust_model].lower() != 'none': galactic_av[i_obj] = float(params[i_gal_dust_model + 1]) galactic_rv[i_obj] = float(params[i_gal_dust_model + 2]) elif params[12].lower() == 'knots': object_type[i_obj] = _RANDOM_WALK semi_major_arcsec[i_obj] = float(params[13]) semi_minor_arcsec[i_obj] = float(params[14]) position_angle_degrees[i_obj] = float(params[15]) npoints[i_obj] = int(params[16]) i_gal_dust_model = 18 if params[17].lower() != 'none': i_gal_dust_model = 20 internal_av[i_obj] = float(params[18]) internal_rv[i_obj] = float(params[19]) if params[i_gal_dust_model].lower() != 'none': galactic_av[i_obj] = float(params[i_gal_dust_model + 1]) galactic_rv[i_obj] = float(params[i_gal_dust_model + 2]) else: raise RuntimeError("Do not know how to handle " "object type: %s" % params[12]) ra_appGeo, dec_appGeo = PhoSimAstrometryBase._appGeoFromPhoSim( np.radians(ra_phosim), np.radians(dec_phosim), obs_md) (ra_obs_rad, dec_obs_rad) = _observedFromAppGeo(ra_appGeo, dec_appGeo, obs_metadata=obs_md, includeRefraction=True) semi_major_radians = radiansFromArcsec(semi_major_arcsec) semi_minor_radians = radiansFromArcsec(semi_minor_arcsec) position_angle_radians = np.radians(position_angle_degrees) x_pupil, y_pupil = _pupilCoordsFromObserved(ra_obs_rad, dec_obs_rad, obs_md) bp_dict = BandpassDict.loadTotalBandpassesFromFiles() sed_dir = lsstUtils.getPackageDir('sims_sed_library') object_is_valid = np.array([True] * num_objects) invalid_objects = np.where( np.logical_or( np.logical_or( mag_norm > 50.0, np.logical_and(galactic_av == 0.0, galactic_rv == 0.0)), np.logical_or( np.logical_and(object_type == _SERSIC_2D, semi_major_arcsec < semi_minor_arcsec), np.logical_and(object_type == _RANDOM_WALK, npoints <= 0)))) object_is_valid[invalid_objects] = False if len(invalid_objects[0]) > 0: message = "\nOmitted %d suspicious objects from " % len( invalid_objects[0]) message += "the instance catalog:\n" n_bad_mag_norm = len(np.where(mag_norm > 50.0)[0]) message += " %d had mag_norm > 50.0\n" % n_bad_mag_norm n_bad_av = len( np.where(np.logical_and(galactic_av == 0.0, galactic_rv == 0.0))[0]) message += " %d had galactic_Av == galactic_Rv == 0\n" % n_bad_av n_bad_axes = len( np.where( np.logical_and(object_type == _SERSIC_2D, semi_major_arcsec < semi_minor_arcsec))[0]) message += " %d had semi_major_axis < semi_minor_axis\n" % n_bad_axes n_bad_knots = len( np.where(np.logical_and(object_type == _RANDOM_WALK, npoints <= 0))[0]) message += " %d had n_points <= 0 \n" % n_bad_knots warnings.warn(message) wav_int = None wav_gal = None gs_object_arr = [] for i_obj in range(num_objects): if not object_is_valid[i_obj]: continue if object_type[i_obj] == _POINT_SOURCE: gs_type = 'pointSource' elif object_type[i_obj] == _SERSIC_2D: gs_type = 'sersic' elif object_type[i_obj] == _RANDOM_WALK: gs_type = 'RandomWalk' # load the SED sed_obj = Sed() sed_obj.readSED_flambda(os.path.join(sed_dir, sed_name[i_obj])) fnorm = getImsimFluxNorm(sed_obj, mag_norm[i_obj]) sed_obj.multiplyFluxNorm(fnorm) if internal_av[i_obj] != 0.0: if wav_int is None or not np.array_equal(sed_obj.wavelen, wav_int): a_int, b_int = sed_obj.setupCCMab() wav_int = copy.deepcopy(sed_obj.wavelen) sed_obj.addCCMDust(a_int, b_int, A_v=internal_av[i_obj], R_v=internal_rv[i_obj]) if redshift[i_obj] != 0.0: sed_obj.redshiftSED(redshift[i_obj], dimming=True) sed_obj.resampleSED(wavelen_match=bp_dict.wavelenMatch) if galactic_av[i_obj] != 0.0: if wav_gal is None or not np.array_equal(sed_obj.wavelen, wav_gal): a_g, b_g = sed_obj.setupCCMab() wav_gal = copy.deepcopy(sed_obj.wavelen) sed_obj.addCCMDust(a_g, b_g, A_v=galactic_av[i_obj], R_v=galactic_rv[i_obj]) gs_object = GalSimCelestialObject(gs_type, x_pupil[i_obj], y_pupil[i_obj], semi_major_radians[i_obj], semi_minor_radians[i_obj], semi_major_radians[i_obj], position_angle_radians[i_obj], sersic_index[i_obj], sed_obj, bp_dict, phot_params, npoints[i_obj], gamma1=gamma1[i_obj], gamma2=gamma2[i_obj], kappa=kappa[i_obj], uniqueId=unique_id[i_obj]) gs_object_arr.append(gs_object) gs_object_arr = np.array(gs_object_arr) # how close to the edge of the detector a source has # to be before we will just simulate it anyway pix_tol = 50.0 # any source brighter than this will be considered # so bright that it should be simulated for all # detectors, just in case light scatters onto them. max_mag = 16.0 # down-select mag_norm, x_pupil, and y_pupil # to only contain those objects that were # deemed to be valid above valid = np.where(object_is_valid) mag_norm = mag_norm[valid] x_pupil = x_pupil[valid] y_pupil = y_pupil[valid] assert len(mag_norm) == len(gs_object_arr) assert len(x_pupil) == len(gs_object_arr) assert len(y_pupil) == len(gs_object_arr) out_obj_dict = {} for det in lsst_camera(): chip_name = det.getName() pixel_corners = getCornerPixels(chip_name, lsst_camera()) x_min = pixel_corners[0][0] x_max = pixel_corners[2][0] y_min = pixel_corners[0][1] y_max = pixel_corners[3][1] xpix, ypix = pixelCoordsFromPupilCoords(x_pupil, y_pupil, chipName=chip_name, camera=lsst_camera()) on_chip = np.where( np.logical_or( mag_norm < max_mag, np.logical_and( xpix > x_min - pix_tol, np.logical_and( xpix < x_max + pix_tol, np.logical_and(ypix > y_min - pix_tol, ypix < y_max + pix_tol))))) out_obj_dict[chip_name] = gs_object_arr[on_chip] return gs_object_arr, out_obj_dict
def _process_chunk(db_lock, log_lock, sema, sed_fit_name, cosmoDC2_data, first_gal, self_dict, bad_gals): """ Do all chunk-specific work: compute table contents for a collection of galaxies and write to db Parameters ---------- db_lock Used to avoid conflicts writing to sqlite output log_lock Used to avoid conflicts writing to per-healpixel log sema A semaphore. Release when done sed_fit_name File where sed fits for this healpixel are cosmoDC2_data Values from cosmoDC2 for this healpixel, keyed by column name first_gal index of first galaxy in our chunk (in sed fit list) self_dict Random useful values stored in GalaxyTruthWriter bad_gals List of galaxy ids, monotone increasing, to be skipped """ dry = self_dict['dry'] chunk_size = self_dict['chunk_size'] dbfile = self_dict['dbfile'] logfile = self_dict['logfile'] if dry: _logit( log_lock, logfile, '_process_chunk invoke for first_gal {}, chunk size {}'.format( first_gal, chunk_size)) if sema is None: return sema.release() #exit(0) return lsst_bp_dict = self_dict['lsst_bp_dict'] galaxy_ids = [] ra = [] dec = [] redshift = [] ebv_vals = None ebv_vals_init = False # does this belong somewhere else? ccm_w = None total_gals = self_dict['total_gals'] chunk_start = first_gal chunk_end = min(first_gal + chunk_size, total_gals) with h5py.File(sed_fit_name, 'r') as sed_fit_file: sed_names = sed_fit_file['sed_names'][()] sed_names = [s.decode() for s in sed_names] # becse stored as bytes gals_this_chunk = chunk_end - chunk_start subset = slice(chunk_start, chunk_end) galaxy_ids = sed_fit_file['galaxy_id'][()][subset] to_log = 'Start with galaxy #{}, id={}\n# galaxies for _process_chunk: {}\n'.format( first_gal, galaxy_ids[0], len(galaxy_ids)) _logit(log_lock, logfile, to_log) # get the cross-match between the sed fit and cosmoDC2 cosmo_len = len(cosmoDC2_data['galaxy_id']) crossmatch_dex = np.searchsorted(cosmoDC2_data['galaxy_id'], galaxy_ids) np.testing.assert_array_equal( galaxy_ids, cosmoDC2_data['galaxy_id'][crossmatch_dex]) ra = sed_fit_file['ra'][()][subset] dec = sed_fit_file['dec'][()][subset] np.testing.assert_array_equal(ra, cosmoDC2_data['ra'][crossmatch_dex]) np.testing.assert_array_equal(dec, cosmoDC2_data['dec'][crossmatch_dex]) good_ixes = _good_indices(galaxy_ids.tolist(), bad_gals[0]) if (len(good_ixes) == 0): if sema is not None: sema.release() return else: _logit( log_lock, logfile, 'Found {} good indices for chunk starting with {}\n'.format( len(good_ixes), chunk_start)) flux_by_band_MW = {} flux_by_band_noMW = {} # Calculate E(B-V) for dust extinction in Milky Way along relevant # lines of sight band_print = "Processing band {}, first gal {}, time {}\n" if not ebv_vals_init: equatorial_coords = np.array([np.radians(ra), np.radians(dec)]) ebv_model = EBVbase() ebv_vals = ebv_model.calculateEbv( equatorialCoordinates=equatorial_coords, interp=True) ebv_vals_init = True for i_bp, bp in enumerate('ugrizy'): if (i_bp == 0 or i_bp == 5): _logit(log_lock, logfile, band_print.format(bp, first_gal, dt.now())) fluxes_noMW = {} fluxes = {} for component in ['disk', 'bulge']: fluxes_noMW[component] = np.zeros(gals_this_chunk, dtype=float) fluxes[component] = np.zeros(gals_this_chunk, dtype=float) for component in ['disk', 'bulge']: #print(" Processing component ", component) sed_arr = sed_fit_file['%s_sed' % component][()][subset] av_arr = sed_fit_file['%s_av' % component][()][subset] rv_arr = sed_fit_file['%s_rv' % component][()][subset] mn_arr = sed_fit_file['%s_magnorm' % component][()][i_bp, :][subset] z_arr = cosmoDC2_data['redshift'][crossmatch_dex] gii = 0 done = False for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate( zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)): if done: break while good_ixes[gii] < i_gal: gii += 1 if gii == len(good_ixes): # ran out of good ones done = True break if done: break if good_ixes[gii] > i_gal: # skipped over it; it's bad continue # Leave space for it in the arrays, but values # for all the fluxes will be left at 0 # read in the SED file from the library sed_file_name = os.path.join(self_dict['sed_lib_dir'], sed_names[s_dex]) sed = sims_photUtils.Sed() sed.readSED_flambda(sed_file_name) # find and apply normalizing flux fnorm = sims_photUtils.getImsimFluxNorm(sed, mn) sed.multiplyFluxNorm(fnorm) # add internal dust if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w): ccm_w = np.copy(sed.wavelen) a_x, b_x = sed.setupCCM_ab() sed.addDust(a_x, b_x, A_v=av, R_v=rv) # apply redshift sed.redshiftSED(zz, dimming=True) # flux, in Janskys, without Milky Way dust extinction f_noMW = sed.calcFlux(lsst_bp_dict[bp]) # apply Milky Way dust # (cannot reuse a_x, b_x because wavelength grid changed # when we called redshiftSED) a_x_mw, b_x_mw = sed.setupCCM_ab() sed.addDust(a_x_mw, b_x_mw, R_v=3.1, ebv=ebv) f_MW = sed.calcFlux(lsst_bp_dict[bp]) fluxes_noMW[component][i_gal] = f_noMW fluxes[component][i_gal] = f_MW if (component == 'disk') and (bp == 'r'): redshift = z_arr # Sum components and convert to nanojansky total_fluxes = (fluxes_noMW['disk'] + fluxes_noMW['bulge']) * 10**9 total_fluxes_MW = (fluxes['disk'] + fluxes['bulge']) * 10**9 dummy_sed = sims_photUtils.Sed() # add magnification due to weak lensing kappa = cosmoDC2_data['convergence'][crossmatch_dex] gamma_sq = (cosmoDC2_data['shear_1'][crossmatch_dex]**2 + cosmoDC2_data['shear_2'][crossmatch_dex]**2) magnification = 1.0 / ((1.0 - kappa)**2 - gamma_sq) magnified_fluxes = magnification * total_fluxes magnified_fluxes_MW = magnification * total_fluxes_MW flux_by_band_noMW[bp] = magnified_fluxes flux_by_band_MW[bp] = magnified_fluxes_MW # Open connection to sqlite db and write #print('Time before db write is {}, first gal={}'.format(dt.now(), first_gal)) #sys.stdout.flush() if not db_lock.acquire(timeout=120.0): _logit(log_lock, logfile, "Failed to acquire db lock, first gal=", first_gal) if sema is None: return sema.release() exit(1) try: _write_sqlite(dbfile, galaxy_ids, ra, dec, redshift, flux_by_band_MW, flux_by_band_noMW, good_ixes) db_lock.release() if sema is not None: sema.release() _logit( log_lock, logfile, 'Time after db write: {}, first_gal={}\n'.format( dt.now(), first_gal)) exit(0) except Exception as ex: db_lock.release() if sema is not None: sema.release() raise (ex)
def validate_sne(cat_dir, obsid, fov_deg=2.1, scatter_file=None, vanishing_file=None): """ Parameters ---------- cat_dir is the parent dir of $obsid obsid is the obsHistID of the pointing (an int) fov_deg is the radius of the field of view in degrees """ project_dir = os.path.join('/global/projecta/projectdirs', 'lsst/groups/SSim/DC2/cosmoDC2_v1.1.4') sne_db_name = os.path.join(project_dir, 'sne_cosmoDC2_v1.1.4_MS_DDF.db') if not os.path.isfile(sne_db_name): raise RuntimeError("\n%s\nis not a file" % sne_db_name) instcat_dir = os.path.join(cat_dir, '%.8d' % obsid) if not os.path.isdir(instcat_dir): raise RuntimeError("\n%s\nis not a dir" % instcat_dir) sne_name = os.path.join(instcat_dir, 'sne_cat_%d.txt.gz' % obsid) if not os.path.isfile(sne_name): raise RuntimeError("\n%s\nis not a file" % sne_name) phosim_name = os.path.join(instcat_dir, 'phosim_cat_%d.txt' % obsid) if not os.path.isfile(phosim_name): raise RuntimeError("\n%s\nis not a file" % phosim_name) sed_dir = os.path.join(instcat_dir, "Dynamic") if not os.path.isdir(sed_dir): raise RuntimeError("\n%s\nis not a dir" % sed_dir) opsim_db = os.path.join("/global/projecta/projectdirs", "lsst/groups/SSim/DC2", "minion_1016_desc_dithered_v4_sfd.db") if not os.path.isfile(opsim_db): raise RuntimeError("\n%s\nis not a file" % opsim_db) band_from_int = 'ugrizy' bandpass = None with open(phosim_name, 'r') as in_file: for line in in_file: params = line.strip().split() if params[0] == 'filter': bandpass = band_from_int[int(params[1])] break if bandpass is None: raise RuntimeError("Failed to read in bandpass") bp_dict = photUtils.BandpassDict.loadTotalBandpassesFromFiles() with sqlite3.connect(opsim_db) as conn: c = conn.cursor() r = c.execute("SELECT descDitheredRA, descDitheredDec, expMJD FROM Summary " "WHERE obsHistID==%d" % obsid).fetchall() pointing_ra = np.degrees(r[0][0]) pointing_dec = np.degrees(r[0][1]) expmjd = float(r[0][2]) with sqlite3.connect(sne_db_name) as conn: c = conn.cursor() query = "SELECT snid_in, snra_in, sndec_in, " query += "c_in, mB, t0_in, x0_in, x1_in, z_in " query += "FROM sne_params WHERE (" where_clause = None half_space = htm.halfSpaceFromRaDec(pointing_ra, pointing_dec, fov_deg) bounds = half_space.findAllTrixels(6) for bound in bounds: if where_clause is not None: where_clause += " OR (" else: where_clause = "(" if bound[0] == bound[1]: where_clause += "htmid_level_6==%d" % bound[0] else: where_clause += "htmid_level_6>=%d AND htmid_level_6<=%d" % (bound[0],bound[1]) where_clause += ")" query += where_clause+")" sn_params = c.execute(query).fetchall() sn_ra = np.array([sn[1] for sn in sn_params]) sn_dec = np.array([sn[2] for sn in sn_params]) sn_d = angularSeparation(pointing_ra, pointing_dec, sn_ra, sn_dec) valid = np.where(sn_d<=fov_deg) sn_ra_dec = {} sn_param_dict = {} for i_sn in valid[0]: sn = sn_params[i_sn] sn_param_dict[sn[0]] = sn[3:] sn_ra_dec[sn[0]] = sn[1:3] sne_in_instcat = set() d_mag_max = -1.0 with gzip.open(sne_name, 'rb') as sne_cat_file: for sne_line in sne_cat_file: instcat_params = sne_line.strip().split(b' ') sne_id = instcat_params[1].decode('utf-8') if sne_id not in sn_param_dict: try: sne_id_int = int(sne_id) assert sne_id_int<1.5e10 except (ValueError, AssertionError): raise RuntimeError("\n%s\nnot in SNe db" % sne_id) sne_in_instcat.add(sne_id) control_params = sn_param_dict[sne_id] sed_name = os.path.join(instcat_dir, instcat_params[5].decode('utf-8')) if not os.path.isfile(sed_name): raise RuntimeError("\n%s\nis not a file" % sed_name) sed = photUtils.Sed() sed.readSED_flambda(sed_name, cache_sed=False) fnorm = photUtils.getImsimFluxNorm(sed, float(instcat_params[4])) sed.multiplyFluxNorm(fnorm) sed.redshiftSED(float(instcat_params[6]), dimming=True) a_x, b_x = sed.setupCCM_ab() A_v = float(instcat_params[15]) R_v = float(instcat_params[16]) EBV = A_v/R_v sed.addDust(a_x, b_x, A_v=A_v, R_v=R_v) sn_object = SNObject() sn_object.set_MWebv(EBV) sn_object.set(c=control_params[0], x1=control_params[4], x0=control_params[3], t0=control_params[2], z=control_params[5]) sn_mag = sn_object.catsimBandMag(bp_dict[bandpass], expmjd) instcat_flux = sed.calcFlux(bp_dict[bandpass]) instcat_mag = sed.magFromFlux(instcat_flux) d_mag = (sn_mag-instcat_mag) if not np.isfinite(d_mag): if np.isfinite(sn_mag) or np.isfinite(instcat_mag): msg = '%s\ngave magnitudes %e %e (diff %e)' % (sne_id, sn_mag, instcat_mag, d_mag) print(msg) if scatter_file is not None: scatter_file.write("%e %e %e\n" % (sn_mag, instcat_mag, d_mag)) d_mag = np.abs(d_mag) if d_mag>d_mag_max: d_mag_max = d_mag #print('dmag %e -- %e (%e)' % # (d_mag, instcat_mag, # control_params[5]-float(instcat_params[6]))) msg = '\n%s\n' % sne_name ct_missing = 0 for sn_id in sn_param_dict: if sn_id in sne_in_instcat: continue control_params = sn_param_dict[sn_id] sn_object = SNObject() sn_object.set_MWebv(0.0) cc = control_params[0] x1 = control_params[4] x0 = control_params[3] t0 = control_params[2] zz = control_params[5] sn_object.set(c=cc, x1=x1, x0=x0, t0=t0, z=zz) sn_mag = sn_object.catsimBandMag(bp_dict[bandpass], expmjd) if vanishing_file is not None and np.isfinite(sn_mag): vanishing_file.write('%d %s %s %e ' % (obsid, bandpass, sn_id, sn_mag)) vanishing_file.write('%e %e %e %.4f %e %.4f %e\n' % (cc,x0,x1,t0,zz,expmjd,expmjd-t0))
def write_sprinkled_lc(out_file_name, total_obs_md, pointing_dir, opsim_db_name, ra_colname='descDitheredRA', dec_colname='descDitheredDec', rottel_colname = 'descDitheredRotTelPos', sql_file_name=None, bp_dict=None): """ Create database of light curves Note: this is still under development. It has not yet been used for a production-level truth catalog Parameters ---------- out_file_name is the name of the sqlite file to be written total_obs_md is an ObservationMetaData covering the whole survey area pointing_dir contains a series of files that are two columns: obshistid, mjd. The files must each have 'visits' in their name. These specify the pointings for which we are assembling data. See: https://github.com/LSSTDESC/DC2_Repo/tree/master/data/Run1.1 for an example. opsim_db_name is the name of the OpSim database to be queried for pointings ra_colname is the column used for RA of the pointing (default: descDitheredRA) dec_colname is the column used for the Dec of the pointing (default: descDitheredDec) rottel_colname is the column used for the rotTelPos of the pointing (default: desckDitheredRotTelPos') sql_file_name is the name of the parameter database produced by write_sprinkled_param_db to be used bp_dict is a BandpassDict of the telescope filters to be used Returns ------- None Writes out a database to out_file_name. The tables of this database and their columns are: light_curves: - uniqueId -- an int unique to all objects - obshistid -- an int unique to all pointings - mag -- the magnitude observed for this object at that pointing obs_metadata: - obshistid -- an int unique to all pointings - mjd -- the date of the pointing - filter -- an int corresponding to the telescope filter (0==u, 1==g..) variables_and_transients: - uniqueId -- an int unique to all objects - galaxy_id -- an int indicating the host galaxy - ra -- in degrees - dec -- in degrees - agn -- ==1 if object is an AGN - sn -- ==1 if object is a supernova """ t0_master = time.time() if not os.path.isfile(sql_file_name): raise RuntimeError('%s does not exist' % sql_file_name) sn_simulator = SneSimulator(bp_dict) sed_dir = os.environ['SIMS_SED_LIBRARY_DIR'] create_sprinkled_sql_file(out_file_name) t_start = time.time() # get data about the pointings being simulated (htmid_dict, mjd_dict, filter_dict, obsmd_dict) = get_pointing_htmid(pointing_dir, opsim_db_name, ra_colname=ra_colname, dec_colname=dec_colname) t_htmid_dict = time.time()-t_start bp_to_int = {'u':0, 'g':1, 'r':2, 'i':3, 'z':4, 'y':5} # put the data about the pointings in the obs_metadata table with sqlite3.connect(out_file_name) as conn: cursor = conn.cursor() values = ((int(obs), mjd_dict[obs], bp_to_int[filter_dict[obs]]) for obs in mjd_dict) cursor.executemany('''INSERT INTO obs_metadata VALUES (?,?,?)''', values) cursor.execute('''CREATE INDEX obs_filter ON obs_metadata (obshistid, filter)''') conn.commit() print('\ngot htmid_dict -- %d in %e seconds' % (len(htmid_dict), t_htmid_dict)) db = DBObject(sql_file_name, driver='sqlite') # get a list of htmid corresponding to trixels in which # variables and transients can be found query = 'SELECT DISTINCT htmid FROM zpoint WHERE is_agn=1 OR is_sn=1' dtype = np.dtype([('htmid', int)]) results = db.execute_arbitrary(query, dtype=dtype) object_htmid = results['htmid'] agn_dtype = np.dtype([('uniqueId', int), ('galaxy_id', int), ('ra', float), ('dec', float), ('redshift', float), ('sed', str, 500), ('magnorm', float), ('varParamStr', str, 500), ('is_sprinkled', int)]) agn_base_query = 'SELECT uniqueId, galaxy_id, ' agn_base_query += 'raJ2000, decJ2000, ' agn_base_query += 'redshift, sedFilepath, ' agn_base_query += 'magNorm, varParamStr, is_sprinkled ' agn_base_query += 'FROM zpoint WHERE is_agn=1 ' sn_dtype = np.dtype([('uniqueId', int), ('galaxy_id', int), ('ra', float), ('dec', float), ('redshift', float), ('sn_truth_params', str, 500), ('is_sprinkled', int)]) sn_base_query = 'SELECT uniqueId, galaxy_id, ' sn_base_query += 'raJ2000, decJ2000, ' sn_base_query += 'redshift, sn_truth_params, is_sprinkled ' sn_base_query += 'FROM zpoint WHERE is_sn=1 ' filter_to_int = {'u':0, 'g':1, 'r':2, 'i':3, 'z':4, 'y':5} n_floats = 0 with sqlite3.connect(out_file_name) as conn: cursor = conn.cursor() t_before_htmid = time.time() # loop over trixels containing variables and transients, simulating # the light curves of those objects for htmid_dex, htmid in enumerate(object_htmid): if htmid_dex>0: htmid_duration = (time.time()-t_before_htmid)/3600.0 htmid_prediction = len(object_htmid)*htmid_duration/htmid_dex print('%d htmid out of %d in %e hours; predict %e hours remaining' % (htmid_dex, len(object_htmid), htmid_duration,htmid_prediction-htmid_duration)) mjd_arr = [] obs_arr = [] filter_arr = [] # Find only those pointings which overlap the current trixel for obshistid in htmid_dict: is_contained = False for bounds in htmid_dict[obshistid]: if htmid<=bounds[1] and htmid>=bounds[0]: is_contained = True break if is_contained: mjd_arr.append(mjd_dict[obshistid]) obs_arr.append(obshistid) filter_arr.append(filter_to_int[filter_dict[obshistid]]) if len(mjd_arr) == 0: continue mjd_arr = np.array(mjd_arr) obs_arr = np.array(obs_arr) filter_arr = np.array(filter_arr) sorted_dex = np.argsort(mjd_arr) mjd_arr = mjd_arr[sorted_dex] obs_arr = obs_arr[sorted_dex] filter_arr = filter_arr[sorted_dex] agn_query = agn_base_query + 'AND htmid=%d' % htmid agn_iter = db.get_arbitrary_chunk_iterator(agn_query, dtype=agn_dtype, chunk_size=10000) # put static data about the AGN (position, etc.) into the # variables_and_transients table for i_chunk, agn_results in enumerate(agn_iter): values = ((int(agn_results['uniqueId'][i_obj]), int(agn_results['galaxy_id'][i_obj]), np.degrees(agn_results['ra'][i_obj]), np.degrees(agn_results['dec'][i_obj]), int(agn_results['is_sprinkled'][i_obj]), 1,0) for i_obj in range(len(agn_results))) cursor.executemany('''INSERT INTO variables_and_transients VALUES (?,?,?,?,?,?,?)''', values) agn_simulator = AgnSimulator(agn_results['redshift']) quiescent_mag = np.zeros((len(agn_results), 6), dtype=float) for i_obj, (sed_name, zz, mm) in enumerate(zip(agn_results['sed'], agn_results['redshift'], agn_results['magnorm'])): spec = Sed() spec.readSED_flambda(os.path.join(sed_dir, sed_name)) fnorm = getImsimFluxNorm(spec, mm) spec.multiplyFluxNorm(fnorm) spec.redshiftSED(zz, dimming=True) mag_list = bp_dict.magListForSed(spec) quiescent_mag[i_obj] = mag_list # simulate AGN variability dmag = agn_simulator.applyVariability(agn_results['varParamStr'], expmjd=mjd_arr) # loop over pointings that overlap the current trixel, writing # out simulated photometry for each AGN observed in that pointing for i_time, obshistid in enumerate(obs_arr): # only include objects that were actually on a detector are_on_chip = _actually_on_chip(np.degrees(agn_results['ra']), np.degrees(agn_results['dec']), obsmd_dict[obshistid]) valid_agn = np.where(are_on_chip) if len(valid_agn[0])==0: continue values = ((int(agn_results['uniqueId'][i_obj]), int(obs_arr[i_time]), quiescent_mag[i_obj][filter_arr[i_time]]+ dmag[filter_arr[i_time]][i_obj][i_time]) for i_obj in valid_agn[0]) cursor.executemany('''INSERT INTO light_curves VALUES (?,?,?)''', values) conn.commit() n_floats += len(dmag.flatten()) sn_query = sn_base_query + 'AND htmid=%d' % htmid sn_iter = db.get_arbitrary_chunk_iterator(sn_query, dtype=sn_dtype, chunk_size=10000) for sn_results in sn_iter: t0_sne = time.time() # write static information about SNe to the # variables_and_transients table values = ((int(sn_results['uniqueId'][i_obj]), int(sn_results['galaxy_id'][i_obj]), np.degrees(sn_results['ra'][i_obj]), np.degrees(sn_results['dec'][i_obj]), int(sn_results['is_sprinkled'][i_obj]), 0,1) for i_obj in range(len(sn_results))) cursor.executemany('''INSERT INTO variables_and_transients VALUES (?,?,?,?,?,?,?)''', values) conn.commit() sn_mags = sn_simulator.calculate_sn_magnitudes(sn_results['sn_truth_params'], mjd_arr, filter_arr) print(' did %d sne in %e seconds' % (len(sn_results), time.time()-t0_sne)) # loop over pointings that overlap the current trixel, writing # out simulated photometry for each SNe observed in that pointing for i_time, obshistid in enumerate(obs_arr): # only include objects that fell on a detector are_on_chip = _actually_on_chip(np.degrees(sn_results['ra']), np.degrees(sn_results['dec']), obsmd_dict[obshistid]) valid_obj = np.where(np.logical_and(np.isfinite(sn_mags[:,i_time]), are_on_chip)) if len(valid_obj[0]) == 0: continue values = ((int(sn_results['uniqueId'][i_obj]), int(obs_arr[i_time]), sn_mags[i_obj][i_time]) for i_obj in valid_obj[0]) cursor.executemany('''INSERT INTO light_curves VALUES (?,?,?)''', values) conn.commit() n_floats += len(valid_obj[0]) cursor.execute('CREATE INDEX unq_obs ON light_curves (uniqueId, obshistid)') conn.commit() print('n_floats %d' % n_floats) print('in %e seconds' % (time.time()-t0_master))
boundType='circle', boundLength=0.1) data_iter = db.query_columns(col_names, obs_metadata=obs, chunk_size=100, constraint="var_type=2") for chunk in data_iter: for star in chunk: sed = Sed() sed.readSED_flambda( os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], defaultSpecMap[star['sedFilename']])) fnorm = getImsimFluxNorm(sed, star['magNorm']) sed.multiplyFluxNorm(fnorm) mags = lsst_bp.magListForSed(sed) du_nodust = np.abs(mags[0] - star['umag']) dg_nodust = np.abs(mags[1] - star['gmag']) dr_nodust = np.abs(mags[2] - star['rmag']) di_nodust = np.abs(mags[3] - star['imag']) dz_nodust = np.abs(mags[4] - star['zmag']) dy_nodust = np.abs(mags[5] - star['ymag']) a_x, b_x = sed.setupCCM_ab() sed.addDust(a_x, b_x, R_v=3.1, ebv=star['ebv']) mags = lsst_bp.magListForSed(sed) du = np.abs(mags[0] - star['umag']) dg = np.abs(mags[1] - star['gmag'])
delta_ast = rng.normal(0.0, ast_err_rad, n_stars) (ra_smeared, dec_smeared) = smear_ra_dec(ra, dec, delta_ast, rng) delta_mag = rng.normal(0.0, phot_err, size=(n_stars, 6)) mags = np.zeros((n_stars, 6), dtype=float) for ii in range(n_stars): spec = photUtils.Sed() file_name = os.path.join(sed_dir, defaultSpecMap[sed_name[ii]]) spec.readSED_flambda(file_name) if not np.array_equal(spec.wavelen, dust_wav): a_x, b_x = spec.setupCCMab() dust_wav = np.copy(spec.wavelen) fnorm = photUtils.getImsimFluxNorm(spec, mag_norm[ii]) spec.multiplyFluxNorm(fnorm) spec.addDust(a_x, b_x, ebv=ebv[ii], R_v=3.1) mags[ii] = bp_dict.magListForSed(spec) for ii in range(n_stars): if mags[ii][2] > r_mag_limit: continue out_file.write('%d, %.10f, %.10f, ' % (unq[ii], ra[ii], dec[ii])) out_file.write('%.12f, %.12f, ' % (ast_err_deg, ast_err_deg)) out_file.write('%.10f, %.10f, ' % (ra_smeared[ii], dec_smeared[ii])) for ibp in range(6): out_file.write('%.5f, %.3f, ' %