def fromThroughputs(cls): """ instantiate class from the LSST throughputs in the throughputs directory """ totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles() return cls(hwBandpassDict=hwbpdict)
def testLoadBandpassesFromFiles(self): """ Test that running the classmethod loadBandpassesFromFiles produces expected result """ fileDir = os.path.join(getPackageDir('sims_photUtils'), 'tests', 'cartoonSedTestData') bandpassNames = ['g', 'z', 'i'] bandpassRoot = 'test_bandpass_' componentList = ['toy_mirror.dat'] atmo = os.path.join(fileDir, 'toy_atmo.dat') bandpassDict, hardwareDict = BandpassDict.loadBandpassesFromFiles(bandpassNames=bandpassNames, filedir=fileDir, bandpassRoot=bandpassRoot, componentList=componentList, atmoTransmission=atmo) controlBandpassList = [] controlHardwareList = [] for bpn in bandpassNames: componentList = [os.path.join(fileDir, bandpassRoot+bpn+'.dat'), os.path.join(fileDir, 'toy_mirror.dat')] dummyBp = Bandpass() dummyBp.readThroughputList(componentList) controlHardwareList.append(dummyBp) componentList = [os.path.join(fileDir, bandpassRoot+bpn+'.dat'), os.path.join(fileDir, 'toy_mirror.dat'), os.path.join(fileDir, 'toy_atmo.dat')] dummyBp = Bandpass() dummyBp.readThroughputList(componentList) controlBandpassList.append(dummyBp) wMin = controlBandpassList[0].wavelen[0] wMax = controlBandpassList[0].wavelen[-1] wStep = controlBandpassList[0].wavelen[1]-controlBandpassList[0].wavelen[0] for bp, hh in zip(controlBandpassList, controlHardwareList): bp.resampleBandpass(wavelen_min=wMin, wavelen_max=wMax, wavelen_step=wStep) hh.resampleBandpass(wavelen_min=wMin, wavelen_max=wMax, wavelen_step=wStep) for test, control in zip(bandpassDict.values(), controlBandpassList): np.testing.assert_array_almost_equal(test.wavelen, control.wavelen, 19) np.testing.assert_array_almost_equal(test.sb, control.sb, 19) for test, control in zip(hardwareDict.values(), controlHardwareList): np.testing.assert_array_almost_equal(test.wavelen, control.wavelen, 19) np.testing.assert_array_almost_equal(test.sb, control.sb, 19)
def fromTwinklesData(cls, tableName, objectTypeID=42, dbHostName=None, idCol='snid', columns=('snid', 'redshift', 'snra', 'sndec', 't0', 'x0', 'x1', 'c'), idSequence=None): """ Simplified classmethod to construct this class from the Twinkles Run 1 perspective. Parameters ---------- tableName : string, mandatory case insensitive string name of table on database to connect to for model parameters of astrophysical objects idCol : string, optional, defaults to 'snid' column name of Index on the table columns : tuple of strings, optional, defaults to values for SN tuple of strings that completely specify the truth values for idSequence : sequence of one dimension, optional, defaults to None sequence of unique ids in the catsim universe indexing the astrophysical objects in the database. dbHostName : string, optional, defaults to None force the class to use this hostname. If not provided, the class will set this to localhost, which is the desired hostname when using an ssh tunnel. This parameter is useful when working from whitelisted computers. Returns ------ An instance of the class RefLightCurve class where the other parameters have been defaulted to sensible values for Twinkles Run1 Analysis. Examples -------- """ data_dir = os.path.join(os.environ['MONITOR_DIR'], 'data') opsimCsv = os.path.join(data_dir, 'SelectedKrakenVisits.csv') opsimdf = pd.read_csv(opsimCsv, index_col='obsHistID') observations = opsimdf[['expMJD', 'filter', 'fiveSigmaDepth']].copy() del opsimdf # Obtain the tuple of total, HardWare bandPassDict and keep the total lsstBP = BandpassDict.loadBandpassesFromFiles()[0] cls = RefLightCurves(tableName=tableName, objectTypeID=objectTypeID, idCol=idCol, dbHostName=dbHostName, columns=columns, observations=observations, bandPassDict=lsstBP, idSequence=idSequence) return cls
def testSignalToNoise(self): """ Test that calcSNR_m5 and calcSNR_sed give similar results """ defaults = LSSTdefaults() photParams = PhotometricParameters() totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles() skySED = Sed() skySED.readSED_flambda( os.path.join(lsst.utils.getPackageDir('throughputs'), 'baseline', 'darksky.dat')) m5 = [] for filt in totalDict: m5.append( calcM5(skySED, totalDict[filt], hardwareDict[filt], photParams, seeing=defaults.seeing(filt))) sedDir = lsst.utils.getPackageDir('sims_sed_library') sedDir = os.path.join(sedDir, 'starSED', 'kurucz') fileNameList = os.listdir(sedDir) numpy.random.seed(42) offset = numpy.random.random_sample(len(fileNameList)) * 2.0 for ix, name in enumerate(fileNameList): if ix > 100: break spectrum = Sed() spectrum.readSED_flambda(os.path.join(sedDir, name)) ff = spectrum.calcFluxNorm(m5[2] - offset[ix], totalDict.values()[2]) spectrum.multiplyFluxNorm(ff) magList = [] controlList = [] magList = [] for filt in totalDict: controlList.append( calcSNR_sed(spectrum, totalDict[filt], skySED, hardwareDict[filt], photParams, defaults.seeing(filt))) magList.append(spectrum.calcMag(totalDict[filt])) testList, gammaList = calcSNR_m5(numpy.array(magList), numpy.array(totalDict.values()), numpy.array(m5), photParams) for tt, cc in zip(controlList, testList): msg = '%e != %e ' % (tt, cc) self.assertTrue(numpy.abs(tt / cc - 1.0) < 0.001, msg=msg)
def __init__(self, idSequence, opsim_csv=None, db_config=None): if opsim_csv is None: opsim_csv = os.path.join(lsst.utils.getPackageDir('monitor'), 'data', 'SelectedKrakenVisits.csv') df = pd.read_csv(opsim_csv, index_col='obsHistID') opsim_df = df[['expMJD', 'filter', 'fiveSigmaDepth']] lsstBP = BandpassDict.loadBandpassesFromFiles()[0] self.reflc = RefLightCurves(idSequence=idSequence, tableName='TwinkSN', bandPassDict=lsstBP, observations=opsim_df)
def testUncertaintyExceptions(self): """ Test that calcSNR_m5 raises exceptions when it needs to """ totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles() magnitudes = numpy.array([22.0, 23.0, 24.0, 25.0, 26.0, 27.0]) shortMagnitudes = numpy.array([22.0]) photParams = PhotometricParameters() shortGamma = numpy.array([1.0, 1.0]) self.assertRaises(RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), shortMagnitudes, photParams) self.assertRaises(RuntimeError, calcSNR_m5, shortMagnitudes, totalDict.values(), magnitudes, photParams) self.assertRaises( RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), magnitudes, photParams, gamma=shortGamma ) snr, gg = calcSNR_m5(magnitudes, totalDict.values(), magnitudes, photParams)
def testSignalToNoise(self): """ Test that calcSNR_m5 and calcSNR_sed give similar results """ defaults = LSSTdefaults() photParams = PhotometricParameters() totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles() skySED = Sed() skySED.readSED_flambda(os.path.join(lsst.utils.getPackageDir("throughputs"), "baseline", "darksky.dat")) m5 = [] for filt in totalDict: m5.append(calcM5(skySED, totalDict[filt], hardwareDict[filt], photParams, seeing=defaults.seeing(filt))) sedDir = lsst.utils.getPackageDir("sims_sed_library") sedDir = os.path.join(sedDir, "starSED", "kurucz") fileNameList = os.listdir(sedDir) numpy.random.seed(42) offset = numpy.random.random_sample(len(fileNameList)) * 2.0 for ix, name in enumerate(fileNameList): if ix > 100: break spectrum = Sed() spectrum.readSED_flambda(os.path.join(sedDir, name)) ff = spectrum.calcFluxNorm(m5[2] - offset[ix], totalDict.values()[2]) spectrum.multiplyFluxNorm(ff) magList = [] controlList = [] magList = [] for filt in totalDict: controlList.append( calcSNR_sed( spectrum, totalDict[filt], skySED, hardwareDict[filt], photParams, defaults.seeing(filt) ) ) magList.append(spectrum.calcMag(totalDict[filt])) testList, gammaList = calcSNR_m5( numpy.array(magList), numpy.array(totalDict.values()), numpy.array(m5), photParams ) for tt, cc in zip(controlList, testList): msg = "%e != %e " % (tt, cc) self.assertTrue(numpy.abs(tt / cc - 1.0) < 0.001, msg=msg)
class TestSkyBrightness(unittest.TestCase): totalbandpassdict, hwbandpassdict = BandpassDict.loadBandpassesFromFiles() skycalc = SkyCalculations(photparams="LSST", hwBandpassDict=hwbandpassdict) def test_skymags(self): skymag = self.skycalc.skymag('g', 0.925184, -0.4789, 61044.077855) assert_almost_equal(skymag, 18.8900, decimal=2) def test_skyDepths(self): m5 = self.skycalc.fiveSigmaDepth('g', 1.086662, 0.925184, -0.4789, 61044.077855, provided_airmass=1.008652, use_provided_airmass=True) assert_almost_equal(m5, 23.0601, decimal=2)
def __init__(self, obs_metadata, photParams, seed=None, bandpassDict=None, addNoise=True, addBackground=True, logger=None): """ @param [in] addNoise is a boolean telling the wrapper whether or not to add noise to the image @param [in] addBackground is a boolean telling the wrapper whether or not to add the skybackground to the image @param [in] seed is an (optional) int that will seed the random number generator used by the noise model. Defaults to None, which causes GalSim to generate the seed from the system. """ self.obs_metadata = obs_metadata self.photParams = photParams if bandpassDict is None: self.bandpassDict = BandpassDict.loadBandpassesFromFiles()[0] # Computing the skybrightness.SkyModel object is expensive, so # do it only once in the constructor. self.skyModel = skybrightness.SkyModel(mags=False) self.addNoise = addNoise self.addBackground = addBackground if logger is not None: self.logger = logger else: self.logger = get_logger('INFO') if seed is None: self.randomNumbers = galsim.UniformDeviate() else: self.randomNumbers = galsim.UniformDeviate(seed)
def testUncertaintyExceptions(self): """ Test that calcSNR_m5 raises exceptions when it needs to """ totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles() magnitudes = numpy.array([22.0, 23.0, 24.0, 25.0, 26.0, 27.0]) shortMagnitudes = numpy.array([22.0]) photParams = PhotometricParameters() shortGamma = numpy.array([1.0, 1.0]) self.assertRaises(RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), shortMagnitudes, photParams) self.assertRaises(RuntimeError, calcSNR_m5, shortMagnitudes, totalDict.values(), magnitudes, photParams) self.assertRaises(RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), magnitudes, photParams, gamma=shortGamma) snr, gg = calcSNR_m5(magnitudes, totalDict.values(), magnitudes, photParams)
import os import numpy as np from lsst.utils import getPackageDir from lsst.sims.catUtils.exampleCatalogDefinitions import DefaultPhoSimHeaderMap from lsst.sims.catUtils.exampleCatalogDefinitions import write_phoSim_header from lsst.sims.catUtils.utils import ObservationMetaDataGenerator from lsst.sims.coordUtils import pupilCoordsFromFocalPlaneCoordsLSST from lsst.sims.utils import raDecFromPupilCoords from lsst.sims.utils import angularSeparation from lsst.sims.utils import Site from lsst.sims.photUtils import BandpassDict, Sed from lsst.sims.utils import altAzPaFromRaDec from lsst.sims.utils import ObservationMetaData bp_dict, hw_dict = BandpassDict.loadBandpassesFromFiles() opsimdb = os.path.join('/Users/danielsf/physics/lsst_150412', 'Development', 'garage', 'OpSimData', 'minion_1016_sqlite.db') obs_gen = ObservationMetaDataGenerator(opsimdb) obs_list = obs_gen.getObservationMetaData(moonAlt=(-90.0, -50.0), altitude=(55.0, 57.0), fieldDec=(-10.0, 10.0)) assert len(obs_list) > 0 obs_root = obs_list[0] obs_root.site = Site(name='LSST', pressure=0.0, humidity=0.0) phosim_header = DefaultPhoSimHeaderMap
def validate_instance_catalog_magnitudes(cat_dir, obsid, seed=99, nrows=-1): """ Parameters ---------- cat_dir is the parent dir of $obsid obsid is the obsHistID of the pointing seed is the seed for a random number generator nrows is the number of galaxies to test (if <0, test all of them) """ agn_dtype = np.dtype([('galaxy_id', int), ('twinkles_id', int)]) agn_cache = np.genfromtxt(os.path.join(os.environ['TWINKLES_DIR'], 'data', 'cosmoDC2_v1.1.4_agn_cache.csv'), dtype=agn_dtype, delimiter=',', skip_header=1) sne_cache = np.genfromtxt(os.path.join(os.environ['TWINKLES_DIR'], 'data', 'cosmoDC2_v1.1.4_sne_cache.csv'), dtype=agn_dtype, delimiter=',', skip_header=1) sprinkled_gid = np.append(agn_cache['galaxy_id'], sne_cache['galaxy_id']) colnames = [ 'obj', 'uniqueID', 'ra', 'dec', 'magnorm', 'sed', 'redshift', 'g1', 'g2', 'kappa', 'dra', 'ddec', 'src_type', 'major', 'minor', 'positionAngle', 'sindex', 'dust_rest', 'rest_av', 'rest_rv', 'dust_obs', 'obs_av', 'obs_rv' ] to_drop = [ 'obj', 'g1', 'g2', 'kappa', 'dra', 'ddec', 'src_type', 'major', 'minor', 'positionAngle', 'sindex', 'dust_rest', 'dust_obs' ] col_types = { 'magnorm': float, 'redshift': float, 'rest_av': float, 'rest_rv': float, 'sed': bytes, 'uniqueID': int } assert os.path.isdir(cat_dir) data_dir = os.path.join(cat_dir, '%.8d' % obsid) if not os.path.isdir(data_dir): raise RuntimeError('\n\n%s\nis not a dir\n\n' % data_dir) phosim_file = os.path.join(data_dir, 'phosim_cat_%d.txt' % obsid) assert os.path.isfile(phosim_file) bandpass_name = None bandpass_name_list = 'ugrizy' with open(phosim_file, 'r') as in_file: for line in in_file: params = line.strip().split() if params[0] == 'filter': bandpass_name = bandpass_name_list[int(params[1])] assert bandpass_name is not None (tot_dict, hw_dict) = BandpassDict.loadBandpassesFromFiles() bandpass = hw_dict[bandpass_name] disk_file = os.path.join(data_dir, 'disk_gal_cat_%d.txt.gz' % obsid) if not os.path.isfile(disk_file): raise RuntimeError("%s is not a file" % disk_file) bulge_file = os.path.join(data_dir, 'bulge_gal_cat_%d.txt.gz' % obsid) assert os.path.isfile(bulge_file) knots_file = os.path.join(data_dir, 'knots_cat_%d.txt.gz' % obsid) assert os.path.isfile(knots_file) print('reading disks') disk_df = pd.read_csv(disk_file, delimiter=' ', compression='gzip', names=colnames, dtype=col_types, nrows=None) disk_df.drop(labels=to_drop, axis='columns', inplace=True) print('read disks') disk_df['galaxy_id'] = pd.Series(disk_df['uniqueID'] // 1024, index=disk_df.index) disk_df = disk_df.set_index('galaxy_id') print('reading bulges') bulge_df = pd.read_csv(bulge_file, delimiter=' ', compression='gzip', names=colnames, dtype=col_types, nrows=None) bulge_df.drop(labels=to_drop, axis='columns', inplace=True) print('read bulges') bulge_df['galaxy_id'] = pd.Series(bulge_df['uniqueID'] // 1024, index=bulge_df.index) bulge_df = bulge_df.set_index('galaxy_id') for ii in range(len(colnames)): colnames[ii] = colnames[ii] + '_knots' for ii in range(len(to_drop)): to_drop[ii] = to_drop[ii] + '_knots' print('reading knots') knots_df = pd.read_csv(knots_file, delimiter=' ', compression='gzip', names=colnames, dtype=col_types, nrows=None) knots_df.drop(labels=to_drop, axis='columns', inplace=True) print('read knots') knots_df['galaxy_id'] = pd.Series(knots_df['uniqueID_knots'] // 1024, index=knots_df.index) knots_df = knots_df.set_index('galaxy_id') wanted_col = [ 'sed', 'magnorm', 'redshift', 'rest_av', 'rest_rv', 'ra', 'dec' ] galaxy_df = disk_df[wanted_col].join(bulge_df[wanted_col], how='outer', lsuffix='_disk', rsuffix='_bulge') for ii in range(len(wanted_col)): wanted_col[ii] = wanted_col[ii] + '_knots' galaxy_df = galaxy_df.join(knots_df[wanted_col], how='outer', rsuffix='_knots') valid_galaxies = np.where( np.logical_not(np.in1d(galaxy_df.index, sprinkled_gid))) galaxy_df = galaxy_df.iloc[valid_galaxies] ra_center = np.nanmedian(galaxy_df['ra_disk'].values) dec_center = np.nanmedian(galaxy_df['dec_disk'].values) dd = angularSeparation(ra_center, dec_center, galaxy_df['ra_disk'].values, galaxy_df['dec_disk'].values) radius_deg = np.nanmax(dd) ra_rad = np.radians(ra_center) dec_rad = np.radians(dec_center) vv = np.array([ np.cos(ra_rad) * np.cos(dec_rad), np.sin(ra_rad) * np.cos(dec_rad), np.sin(dec_rad) ]) healpix_list = healpy.query_disc(32, vv, np.radians(radius_deg), nest=False, inclusive=True) gal_id_values = galaxy_df.index.values cat = GCRCatalogs.load_catalog('cosmoDC2_v1.1.4_image') cat_qties = {} cat_qties['galaxy_id'] = [] cat_qties['ra'] = [] cat_qties['dec'] = [] for hp in healpix_list: hp_query = GCRQuery('healpix_pixel==%d' % hp) local_qties = cat.get_quantities(['galaxy_id', 'ra', 'dec'], native_filters=[hp_query]) valid = np.in1d(local_qties['galaxy_id'], gal_id_values) if valid.any(): for k in local_qties: cat_qties[k].append(local_qties[k][valid]) for k in cat_qties: cat_qties[k] = np.concatenate(cat_qties[k]) cat_dexes = np.arange(len(cat_qties['galaxy_id']), dtype=int) if nrows > 0: rng = np.random.RandomState(seed) dexes = rng.choice(galaxy_df.index.values, size=nrows, replace=False) galaxy_df = galaxy_df.loc[dexes] galaxy_df = galaxy_df.sort_index() invalid_knots = np.where( np.logical_not( np.isfinite(galaxy_df['magnorm_knots'].values.astype(np.float)))) dd = angularSeparation(ra_center, dec_center, cat_qties['ra'], cat_qties['dec']) dd_cut = np.where(dd < (radius_deg + 0.05)) gid = cat_qties['galaxy_id'][dd_cut] cat_dexes = cat_dexes[dd_cut] in1d_valid_dexes = np.where( np.in1d(gid, galaxy_df.index.values, assume_unique=True)) valid_dexes = cat_dexes[in1d_valid_dexes] gid = gid[in1d_valid_dexes] sorted_dex = np.argsort(gid) valid_dexes = valid_dexes[sorted_dex] assert len(gid) == len(galaxy_df.index.values) np.testing.assert_array_equal(gid[sorted_dex], galaxy_df.index.values) mag_name = 'mag_true_%s_lsst' % bandpass_name qties = {} qties['galaxy_id'] = [] qties[mag_name] = [] for hp in healpix_list: hp_query = GCRQuery('healpix_pixel==%d' % hp) local_qties = cat.get_quantities(['galaxy_id', mag_name], native_filters=[hp_query]) valid = np.in1d(local_qties['galaxy_id'], gal_id_values) if valid.any(): for k in local_qties: qties[k].append(local_qties[k][valid]) for k in qties: qties[k] = np.concatenate(qties[k]) np.testing.assert_array_equal(qties['galaxy_id'], cat_qties['galaxy_id']) mags = qties[mag_name][valid_dexes] gid = qties['galaxy_id'][valid_dexes] assert len(gid) == len(mags) assert len(mags) > 0 if nrows > 0: assert len(mags) == nrows t_start = time.time() n_proc = 3 d_proc = len(gid) // n_proc mgr = multiprocessing.Manager() out_dict = mgr.dict() p_list = [] for i_start in range(0, len(gid), d_proc): mag_true = mags[i_start:i_start + d_proc] galaxy_arr = galaxy_df.iloc[i_start:i_start + d_proc] p = multiprocessing.Process(target=validate_batch, args=(mag_true, galaxy_arr, bandpass, out_dict)) p.start() p_list.append(p) for p in p_list: p.join() assert len(list(out_dict.keys())) > 0 d_mag_max = 0.0 for k in out_dict.keys(): if out_dict[k] > d_mag_max: d_mag_max = out_dict[k] if d_mag_max > 1.0e-5: raise RuntimeError("\nobsHistID failed magnitud validation\n" "d_mag_max %e" % d_mag_max)
def sed_from_galacticus_mags(galacticus_mags, redshift, redshift_true, H0, Om0, wav_min, wav_width, obs_lsst_mags): """ Fit SEDs from sims_sed_library to Galacticus galaxies based on the magnitudes in tophat filters. Parameters ---------- galacticus_mags is a numpy array such that galacticus_mags[i][j] is the magnitude of the jth star in the ith bandpass, where the bandpasses are ordered in ascending order of minimum wavelength. redshift is an array of redshifts for the galaxies being fit (includes cosmology and proper motion) redshift_true is an array of cosmological redshifts for the galaxies being fit H0 is the Hubbleparameter in units of km/s/Mpc Om0 is the critical density parameter for matter wav_min is a numpy array of the minimum wavelengths of the tophat filters (in nm) wav_grid is a numpy array of the widths of the tophat filters (in nm) ob_lsst_mags is a numpy array of observer frame LSST magnitudes. obs_lsst_mags[0] will contain the u band magnitudes of every object. Returns ------- a numpy array of SED names and a numpy array of magNorms. """ if (not hasattr(sed_from_galacticus_mags, '_color_tree') or not np.allclose(wav_min, sed_from_galacticus_mags._wav_min, atol=1.0e-10, rtol=0.0) or not np.allclose(wav_width, sed_from_galacticus_mags._wav_width, atol=1.0e-10, rtol=0.0)): (sed_names, sed_mag_list, sed_mag_norm, av_grid, rv_grid) = _create_sed_library_mags(wav_min, wav_width) assert rv_grid.min() > 0.0 assert len(np.where(np.logical_not(np.isfinite(rv_grid)))[0]) == 0 sed_colors = sed_mag_list[:, 1:] - sed_mag_list[:, :-1] sed_from_galacticus_mags._sed_names = sed_names sed_from_galacticus_mags._mag_norm = sed_mag_norm # N_sed sed_from_galacticus_mags._av_grid = av_grid sed_from_galacticus_mags._rv_grid = rv_grid sed_from_galacticus_mags._sed_mags = sed_mag_list # N_sed by N_mag sed_from_galacticus_mags._color_tree = scipy_spatial.cKDTree( sed_colors) sed_from_galacticus_mags._wav_min = wav_min sed_from_galacticus_mags._wav_width = wav_width if (not hasattr(sed_from_galacticus_mags, '_cosmo') or np.abs(sed_from_galacticus_mags._cosmo.H() - H0) > 1.0e-6 or np.abs(sed_from_galacticus_mags._cosmo.OmegaMatter() - Om0) > 1.0e-6): sed_from_galacticus_mags._cosmo = CosmologyObject(H0=H0, Om0=Om0) galacticus_mags_t = np.asarray(galacticus_mags).T # N_star by N_mag assert galacticus_mags_t.shape == ( len(redshift), sed_from_galacticus_mags._sed_mags.shape[1]) with np.errstate(invalid='ignore', divide='ignore'): galacticus_colors = galacticus_mags_t[:, 1:] - galacticus_mags_t[:, : -1] # N_star by (N_mag - 1) t_start = time.time() (sed_dist, sed_idx) = sed_from_galacticus_mags._color_tree.query(galacticus_colors, k=1) # cKDTree returns an invalid index (==len(tree_data)) in cases # where the distance is not finite sed_idx = np.where(sed_idx < len(sed_from_galacticus_mags._sed_names), sed_idx, 0) distance_modulus = sed_from_galacticus_mags._cosmo.distanceModulus( redshift=redshift_true) output_names = sed_from_galacticus_mags._sed_names[sed_idx] (lsst_bp_dict, dummy_bp_dict) = BandpassDict.loadBandpassesFromFiles() output_mag_norm = np.zeros((6, len(output_names)), dtype=float) base_norm = sed_from_galacticus_mags._mag_norm[sed_idx] assert len(np.where(np.logical_not(np.isfinite(base_norm)))[0]) == 0 ccm_w = None av_arr = sed_from_galacticus_mags._av_grid[sed_idx] rv_arr = sed_from_galacticus_mags._rv_grid[sed_idx] assert rv_arr.min() > 0.0 assert len(np.where(np.logical_not(np.isfinite(rv_arr)))[0]) == 0 for i_bp in range(6): output_mag_norm[i_bp, :] = base_norm + distance_modulus sed_dir = getPackageDir('sims_sed_library') for i_obj in range(len(output_names)): spec = Sed() spec.readSED_flambda(os.path.join(sed_dir, output_names[i_obj])) if ccm_w is None or not np.array_equal(spec.wavelen, ccm_w): ccm_w = np.copy(spec.wavelen) ax, bx = spec.setupCCM_ab() spec.addDust(ax, bx, A_v=av_arr[i_obj], R_v=rv_arr[i_obj]) spec.redshiftSED(redshift[i_obj], dimming=True) lsst_mags = lsst_bp_dict.magListForSed(spec) d_mag = obs_lsst_mags[:, i_obj] - lsst_mags output_mag_norm[:, i_obj] += d_mag return (output_names, output_mag_norm, av_arr, rv_arr)
# Bandpasses and LSST related stuff def atmTransName(airmass): """ obtain the filename with the transmissions for files for an airmass value closest to the requested value """ l = np.arange(1.0, 2.51, 0.1) idx = np.abs(l - airmass).argmin() a = np.int(10 * l[idx]) baseline = getPackageDir('THROUGHPUTS') fname = os.path.join(baseline, 'atmos', 'atmos_{}_aerosol.dat'.format(a)) return fname logger.info('Get Bandpasses') totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles() photparams = PhotometricParameters() # Split the entries in the opsim database for parallelization splits = 40 dfs = np.array_split(df, splits) print( 'splitting dataframe of size {0} into {1} splits each of size {2}'.format( len(df), splits, len(dfs[0]))) calcdfs = [] def recalcmags(j, lst=calcdfs): logfname = 'newres_{}.log'.format(j) with open(logfname, 'w') as f:
print('numpy version: ', np.__version__) print('healpy dir', getPackageDir('healpy')) print('sims_skybrightness_dir', getPackageDir('sims_skybrightness')) print('sims_skybrightness_data_dir', getPackageDir('sims_skybrightness_data')) print('obscond version', obscond.__version__) logger.info('Start reading opsim database') minion_out = '/local/lsst/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db' #minion_out = '/Users/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db' opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="unique_all") # opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="ddf") print('reading done\n') df = opsout.summary.copy() logger.info('Finished reading database at {}'.format(time.time())) totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles() photparams = PhotometricParameters() # Split the entries in the opsim database for parallelization splits = 1000 dfs = np.array_split(df, splits) print('splitting dataframe of size {0} into {1} splits each of size {2}\n'.format(len(df), splits, len(dfs[0]))) calcdfs = [] def recalcmags(j): logfname = 'newres_{}.log'.format(j) tsplitstart = time.time() with open(logfname, 'w') as f: f.write('starting split {0} at time {1}\n'.format(j, tsplitstart)) df = dfs[j]
def _initializeGalSimInterpreter(self): """ This method creates the GalSimInterpreter (if it is None) This method reads in all of the data about the camera and pass it into the GalSimInterpreter. This method calls _getBandpasses to construct the paths to the files containing the bandpass data. """ if not isinstance(self.camera_wrapper, GalSimCameraWrapper): raise RuntimeError("GalSimCatalog.camera_wrapper must be an instantiation of " "GalSimCameraWrapper or one of its daughter classes\n" "It is actually of type %s" % str(type(self.camera_wrapper))) if self.galSimInterpreter is None: # This list will contain instantiations of the GalSimDetector class # (see galSimInterpreter.py), which stores detector information in a way # that the GalSimInterpreter will understand detectors = [] for dd in self.camera_wrapper.camera: if dd.getType() == WAVEFRONT or dd.getType() == GUIDER: # This package does not yet handle the 90-degree rotation # in WCS that occurs for wavefront or guide sensors continue if self.allowed_chips is None or dd.getName() in self.allowed_chips: centerPupil = self.camera_wrapper.getCenterPupil(dd.getName()) centerPixel = self.camera_wrapper.getCenterPixel(dd.getName()) translationPupil = self.camera_wrapper.pupilCoordsFromPixelCoords(centerPixel.getX()+1, centerPixel.getY()+1, dd.getName()) plateScale = np.sqrt(np.power(translationPupil[0]-centerPupil.getX(), 2) + np.power(translationPupil[1]-centerPupil.getY(), 2))/np.sqrt(2.0) plateScale = 3600.0*np.degrees(plateScale) # make a detector-custom photParams that copies all of the quantities # in the catalog photParams, except the platescale, which is # calculated above params = PhotometricParameters(exptime=self.photParams.exptime, nexp=self.photParams.nexp, effarea=self.photParams.effarea, gain=self.photParams.gain, readnoise=self.photParams.readnoise, darkcurrent=self.photParams.darkcurrent, othernoise=self.photParams.othernoise, platescale=plateScale) detector = GalSimDetector(dd.getName(), self.camera_wrapper, obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, photParams=params) detectors.append(detector) if not hasattr(self, 'bandpassDict'): if self.noise_and_background is not None: if self.obs_metadata.m5 is None: raise RuntimeError('WARNING in GalSimCatalog; you did not specify m5 in your ' 'obs_metadata. m5 is required in order to ' 'add noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.m5: raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' + 'm5 values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'm5 has: %s ' % list(self.obs_metadata.m5.keys()).__repr__()) if self.obs_metadata.seeing is None: raise RuntimeError('WARNING in GalSimCatalog; you did not specify seeing in your ' 'obs_metadata. seeing is required in order to add ' 'noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.seeing: raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' + 'seeing values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'seeing has: %s ' % list(self.obs_metadata.seeing.keys()).__repr__()) (self.bandpassDict, hardwareDict) = BandpassDict.loadBandpassesFromFiles(bandpassNames=self.bandpassNames, filedir=self.bandpassDir, bandpassRoot=self.bandpassRoot, componentList=self.componentList, atmoTransmission=os.path.join(self.bandpassDir, self.atmoTransmissionName)) self.galSimInterpreter = GalSimInterpreter(obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, detectors=detectors, bandpassDict=self.bandpassDict, noiseWrapper=self.noise_and_background, seed=self.seed) self.galSimInterpreter.setPSF(PSF=self.PSF)
def _initializeGalSimInterpreter(self): """ This method creates the GalSimInterpreter (if it is None) This method reads in all of the data about the camera and pass it into the GalSimInterpreter. This method calls _getBandpasses to construct the paths to the files containing the bandpass data. """ if not isinstance(self.camera_wrapper, GalSimCameraWrapper): raise RuntimeError( "GalSimCatalog.camera_wrapper must be an instantiation of " "GalSimCameraWrapper or one of its daughter classes\n" "It is actually of type %s" % str(type(self.camera_wrapper))) if self.galSimInterpreter is None: # This list will contain instantiations of the GalSimDetector class # (see galSimInterpreter.py), which stores detector information in a way # that the GalSimInterpreter will understand detectors = [] for dd in self.camera_wrapper.camera: if dd.getType() == WAVEFRONT or dd.getType() == GUIDER: # This package does not yet handle the 90-degree rotation # in WCS that occurs for wavefront or guide sensors continue if self.allowed_chips is None or dd.getName( ) in self.allowed_chips: detectors.append( make_galsim_detector(self.camera_wrapper, dd.getName(), self.photParams, self.obs_metadata, epoch=self.db_obj.epoch)) if not hasattr(self, 'bandpassDict'): if self.noise_and_background is not None: if self.obs_metadata.m5 is None: raise RuntimeError( 'WARNING in GalSimCatalog; you did not specify m5 in your ' 'obs_metadata. m5 is required in order to ' 'add noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.m5: raise RuntimeError( 'WARNING in GalSimCatalog; your obs_metadata does not have ' + 'm5 values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'm5 has: %s ' % list(self.obs_metadata.m5.keys()).__repr__()) if self.obs_metadata.seeing is None: raise RuntimeError( 'WARNING in GalSimCatalog; you did not specify seeing in your ' 'obs_metadata. seeing is required in order to add ' 'noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.seeing: raise RuntimeError( 'WARNING in GalSimCatalog; your obs_metadata does not have ' + 'seeing values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'seeing has: %s ' % list(self.obs_metadata.seeing.keys()).__repr__( )) (self.bandpassDict, hardwareDict) = BandpassDict.loadBandpassesFromFiles( bandpassNames=self.bandpassNames, filedir=self.bandpassDir, bandpassRoot=self.bandpassRoot, componentList=self.componentList, atmoTransmission=os.path.join(self.bandpassDir, self.atmoTransmissionName)) self.galSimInterpreter = GalSimInterpreter( obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, detectors=detectors, bandpassDict=self.bandpassDict, noiseWrapper=self.noise_and_background, seed=self.seed) self.galSimInterpreter.setPSF(PSF=self.PSF)
t_start = time.time() sed_name_list, mag_norm_list = sed_from_galacticus_mags(disk_mags, true_redshift_list) rest_sed_name_list, rest_mag_norm_list=sed_from_galacticus_mags(disk_mags, np.zeros(len(true_redshift_list), dtype=float)) np.testing.assert_array_equal(sed_name_list, rest_sed_name_list) print("fitting %d took %.3e" % (len(sed_name_list), time.time()-t_start)) print("mag norm %e %e %e" % (mag_norm_list.min(), np.median(mag_norm_list), mag_norm_list.max())) assert len(sed_name_list) == len(first_disk) sed_dir = getPackageDir('sims_sed_library') gal_sed_dir = os.path.join(sed_dir, 'galaxySED') from lsst.sims.photUtils import Sed, BandpassDict, getImsimFluxNorm total_bp_dict, lsst_bp_dict = BandpassDict.loadBandpassesFromFiles() worst_dist = -1.0 av_valid = np.where(np.logical_and(av_list>0.01, np.logical_and(ebv_list>0.0, av_list<5.0))) sed_name_list = sed_name_list[av_valid] mag_norm_list = mag_norm_list[av_valid] rest_mag_norm_list = rest_mag_norm_list[av_valid] true_redshift_list = true_redshift_list[av_valid] full_redshift_list = full_redshift_list[av_valid] av_list = av_list[av_valid] ebv_list = ebv_list[av_valid] u_control = u_control[av_valid] g_control = g_control[av_valid] r_control = r_control[av_valid]
def _initializeGalSimInterpreter(self): """ This method creates the GalSimInterpreter (if it is None) This method reads in all of the data about the camera and pass it into the GalSimInterpreter. This method calls _getBandpasses to construct the paths to the files containing the bandpass data. """ if self.galSimInterpreter is None: # This list will contain instantiations of the GalSimDetector class # (see galSimInterpreter.py), which stores detector information in a way # that the GalSimInterpreter will understand detectors = [] for dd in self.camera: if self.allowed_chips is None or dd.getName( ) in self.allowed_chips: cs = dd.makeCameraSys(PUPIL) centerPupil = self.camera.transform( dd.getCenter(FOCAL_PLANE), cs).getPoint() centerPixel = dd.getCenter(PIXELS).getPoint() translationPixel = afwGeom.Point2D(centerPixel.getX() + 1, centerPixel.getY() + 1) translationPupil = self.camera.transform( dd.makeCameraPoint(translationPixel, PIXELS), cs).getPoint() plateScale = np.sqrt( np.power(translationPupil.getX() - centerPupil.getX(), 2) + np.power(translationPupil.getY() - centerPupil.getY(), 2)) / np.sqrt(2.0) plateScale = 3600.0 * np.degrees(plateScale) # make a detector-custom photParams that copies all of the quantities # in the catalog photParams, except the platescale, which is # calculated above params = PhotometricParameters( exptime=self.photParams.exptime, nexp=self.photParams.nexp, effarea=self.photParams.effarea, gain=self.photParams.gain, readnoise=self.photParams.readnoise, darkcurrent=self.photParams.darkcurrent, othernoise=self.photParams.othernoise, platescale=plateScale) detector = GalSimDetector(dd, self.camera, obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, photParams=params) detectors.append(detector) if not hasattr(self, 'bandpassDict'): if self.noise_and_background is not None: if self.obs_metadata.m5 is None: raise RuntimeError( 'WARNING in GalSimCatalog; you did not specify m5 in your ' 'obs_metadata. m5 is required in order to ' 'add noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.m5: raise RuntimeError( 'WARNING in GalSimCatalog; your obs_metadata does not have ' + 'm5 values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'm5 has: %s ' % list(self.obs_metadata.m5.keys()).__repr__()) if self.obs_metadata.seeing is None: raise RuntimeError( 'WARNING in GalSimCatalog; you did not specify seeing in your ' 'obs_metadata. seeing is required in order to add ' 'noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.seeing: raise RuntimeError( 'WARNING in GalSimCatalog; your obs_metadata does not have ' + 'seeing values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'seeing has: %s ' % list(self.obs_metadata.seeing.keys()).__repr__( )) (self.bandpassDict, hardwareDict) = BandpassDict.loadBandpassesFromFiles( bandpassNames=self.bandpassNames, filedir=self.bandpassDir, bandpassRoot=self.bandpassRoot, componentList=self.componentList, atmoTransmission=os.path.join(self.bandpassDir, self.atmoTransmissionName)) self.galSimInterpreter = GalSimInterpreter( obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, detectors=detectors, bandpassDict=self.bandpassDict, noiseWrapper=self.noise_and_background, seed=self.seed) self.galSimInterpreter.setPSF(PSF=self.PSF)
def _initializeGalSimInterpreter(self): """ This method creates the GalSimInterpreter (if it is None) This method reads in all of the data about the camera and pass it into the GalSimInterpreter. This method calls _getBandpasses to construct the paths to the files containing the bandpass data. """ if self.galSimInterpreter is None: #This list will contain instantiations of the GalSimDetector class #(see galSimInterpreter.py), which stores detector information in a way #that the GalSimInterpreter will understand detectors = [] for dd in self.camera: if self.allowed_chips is None or dd.getName() in self.allowed_chips: cs = dd.makeCameraSys(PUPIL) centerPupil = self.camera.transform(dd.getCenter(FOCAL_PLANE),cs).getPoint() centerPixel = dd.getCenter(PIXELS).getPoint() translationPixel = afwGeom.Point2D(centerPixel.getX()+1, centerPixel.getY()+1) translationPupil = self.camera.transform( dd.makeCameraPoint(translationPixel, PIXELS), cs).getPoint() plateScale = numpy.sqrt(numpy.power(translationPupil.getX()-centerPupil.getX(),2)+ numpy.power(translationPupil.getY()-centerPupil.getY(),2))/numpy.sqrt(2.0) plateScale = 3600.0*numpy.degrees(plateScale) #make a detector-custom photParams that copies all of the quantities #in the catalog photParams, except the platescale, which is #calculated above params = PhotometricParameters(exptime=self.photParams.exptime, nexp=self.photParams.nexp, effarea=self.photParams.effarea, gain=self.photParams.gain, readnoise=self.photParams.readnoise, darkcurrent=self.photParams.darkcurrent, othernoise=self.photParams.othernoise, platescale=plateScale) detector = GalSimDetector(dd, self.camera, obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, photParams=params) detectors.append(detector) if not hasattr(self, 'bandpassDict'): if self.noise_and_background is not None: if self.obs_metadata.m5 is None: raise RuntimeError('WARNING in GalSimCatalog; you did not specify m5 in your '+ 'obs_metadata. m5 is required in order to add noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.m5: raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' + 'm5 values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'm5 has: %s ' % self.obs_metadata.m5.keys().__repr__()) if self.obs_metadata.seeing is None: raise RuntimeError('WARNING in GalSimCatalog; you did not specify seeing in your '+ 'obs_metadata. seeing is required in order to add noise to your images') for name in self.bandpassNames: if name not in self.obs_metadata.seeing: raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' + 'seeing values for all of your bandpasses \n' + 'bandpass has: %s \n' % self.bandpassNames.__repr__() + 'seeing has: %s ' % self.obs_metadata.seeing.keys().__repr__()) self.bandpassDict, hardwareDict = BandpassDict.loadBandpassesFromFiles(bandpassNames=self.bandpassNames, filedir=self.bandpassDir, bandpassRoot=self.bandpassRoot, componentList=self.componentList, atmoTransmission=os.path.join(self.bandpassDir, self.atmoTransmissionName)) self.galSimInterpreter = GalSimInterpreter(obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, detectors=detectors, bandpassDict=self.bandpassDict, noiseWrapper=self.noise_and_background, seed=self.seed) self.galSimInterpreter.setPSF(PSF=self.PSF)
def __init__(self): self.lsst_bandpasses = BandpassDict.loadBandpassesFromFiles()[0] self._gs_bandpasses = dict()