def fromThroughputs(cls):
"""
instantiate class from the LSST throughputs in the throughputs
directory
"""
totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles()
return cls(hwBandpassDict=hwbpdict)
def testLoadBandpassesFromFiles(self):
"""
Test that running the classmethod loadBandpassesFromFiles produces
expected result
"""
fileDir = os.path.join(getPackageDir('sims_photUtils'),
'tests', 'cartoonSedTestData')
bandpassNames = ['g', 'z', 'i']
bandpassRoot = 'test_bandpass_'
componentList = ['toy_mirror.dat']
atmo = os.path.join(fileDir, 'toy_atmo.dat')
bandpassDict, hardwareDict = BandpassDict.loadBandpassesFromFiles(bandpassNames=bandpassNames,
filedir=fileDir,
bandpassRoot=bandpassRoot,
componentList=componentList,
atmoTransmission=atmo)
controlBandpassList = []
controlHardwareList = []
for bpn in bandpassNames:
componentList = [os.path.join(fileDir, bandpassRoot+bpn+'.dat'),
os.path.join(fileDir, 'toy_mirror.dat')]
dummyBp = Bandpass()
dummyBp.readThroughputList(componentList)
controlHardwareList.append(dummyBp)
componentList = [os.path.join(fileDir, bandpassRoot+bpn+'.dat'),
os.path.join(fileDir, 'toy_mirror.dat'),
os.path.join(fileDir, 'toy_atmo.dat')]
dummyBp = Bandpass()
dummyBp.readThroughputList(componentList)
controlBandpassList.append(dummyBp)
wMin = controlBandpassList[0].wavelen[0]
wMax = controlBandpassList[0].wavelen[-1]
wStep = controlBandpassList[0].wavelen[1]-controlBandpassList[0].wavelen[0]
for bp, hh in zip(controlBandpassList, controlHardwareList):
bp.resampleBandpass(wavelen_min=wMin, wavelen_max=wMax,
wavelen_step=wStep)
hh.resampleBandpass(wavelen_min=wMin, wavelen_max=wMax,
wavelen_step=wStep)
for test, control in zip(bandpassDict.values(), controlBandpassList):
np.testing.assert_array_almost_equal(test.wavelen,
control.wavelen, 19)
np.testing.assert_array_almost_equal(test.sb, control.sb, 19)
for test, control in zip(hardwareDict.values(), controlHardwareList):
np.testing.assert_array_almost_equal(test.wavelen,
control.wavelen, 19)
np.testing.assert_array_almost_equal(test.sb, control.sb, 19)
def fromTwinklesData(cls,
tableName,
objectTypeID=42,
dbHostName=None,
idCol='snid',
columns=('snid', 'redshift', 'snra', 'sndec', 't0',
'x0', 'x1', 'c'),
idSequence=None):
"""
Simplified classmethod to construct this class from the Twinkles Run 1
perspective.
Parameters
----------
tableName : string, mandatory
case insensitive string name of table on database to connect to
for model parameters of astrophysical objects
idCol : string, optional, defaults to 'snid'
column name of Index on the table
columns : tuple of strings, optional, defaults to values for SN
tuple of strings that completely specify the truth values for
idSequence : sequence of one dimension, optional, defaults to None
sequence of unique ids in the catsim universe indexing the
astrophysical objects in the database.
dbHostName : string, optional, defaults to None
force the class to use this hostname. If not provided, the class
will set this to localhost, which is the desired hostname when
using an ssh tunnel. This parameter is useful when working from
whitelisted computers.
Returns
------
An instance of the class RefLightCurve class where the other parameters
have been defaulted to sensible values for Twinkles Run1 Analysis.
Examples
--------
"""
data_dir = os.path.join(os.environ['MONITOR_DIR'], 'data')
opsimCsv = os.path.join(data_dir, 'SelectedKrakenVisits.csv')
opsimdf = pd.read_csv(opsimCsv, index_col='obsHistID')
observations = opsimdf[['expMJD', 'filter', 'fiveSigmaDepth']].copy()
del opsimdf
# Obtain the tuple of total, HardWare bandPassDict and keep the total
lsstBP = BandpassDict.loadBandpassesFromFiles()[0]
cls = RefLightCurves(tableName=tableName,
objectTypeID=objectTypeID,
idCol=idCol,
dbHostName=dbHostName,
columns=columns,
observations=observations,
bandPassDict=lsstBP,
idSequence=idSequence)
return cls
def testSignalToNoise(self):
"""
Test that calcSNR_m5 and calcSNR_sed give similar results
"""
defaults = LSSTdefaults()
photParams = PhotometricParameters()
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
skySED = Sed()
skySED.readSED_flambda(
os.path.join(lsst.utils.getPackageDir('throughputs'), 'baseline',
'darksky.dat'))
m5 = []
for filt in totalDict:
m5.append(
calcM5(skySED,
totalDict[filt],
hardwareDict[filt],
photParams,
seeing=defaults.seeing(filt)))
sedDir = lsst.utils.getPackageDir('sims_sed_library')
sedDir = os.path.join(sedDir, 'starSED', 'kurucz')
fileNameList = os.listdir(sedDir)
numpy.random.seed(42)
offset = numpy.random.random_sample(len(fileNameList)) * 2.0
for ix, name in enumerate(fileNameList):
if ix > 100:
break
spectrum = Sed()
spectrum.readSED_flambda(os.path.join(sedDir, name))
ff = spectrum.calcFluxNorm(m5[2] - offset[ix],
totalDict.values()[2])
spectrum.multiplyFluxNorm(ff)
magList = []
controlList = []
magList = []
for filt in totalDict:
controlList.append(
calcSNR_sed(spectrum, totalDict[filt], skySED,
hardwareDict[filt], photParams,
defaults.seeing(filt)))
magList.append(spectrum.calcMag(totalDict[filt]))
testList, gammaList = calcSNR_m5(numpy.array(magList),
numpy.array(totalDict.values()),
numpy.array(m5), photParams)
for tt, cc in zip(controlList, testList):
msg = '%e != %e ' % (tt, cc)
self.assertTrue(numpy.abs(tt / cc - 1.0) < 0.001, msg=msg)
def __init__(self, idSequence, opsim_csv=None, db_config=None):
if opsim_csv is None:
opsim_csv = os.path.join(lsst.utils.getPackageDir('monitor'),
'data', 'SelectedKrakenVisits.csv')
df = pd.read_csv(opsim_csv, index_col='obsHistID')
opsim_df = df[['expMJD', 'filter', 'fiveSigmaDepth']]
lsstBP = BandpassDict.loadBandpassesFromFiles()[0]
self.reflc = RefLightCurves(idSequence=idSequence,
tableName='TwinkSN',
bandPassDict=lsstBP,
observations=opsim_df)
def testUncertaintyExceptions(self):
"""
Test that calcSNR_m5 raises exceptions when it needs to
"""
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
magnitudes = numpy.array([22.0, 23.0, 24.0, 25.0, 26.0, 27.0])
shortMagnitudes = numpy.array([22.0])
photParams = PhotometricParameters()
shortGamma = numpy.array([1.0, 1.0])
self.assertRaises(RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), shortMagnitudes, photParams)
self.assertRaises(RuntimeError, calcSNR_m5, shortMagnitudes, totalDict.values(), magnitudes, photParams)
self.assertRaises(
RuntimeError, calcSNR_m5, magnitudes, totalDict.values(), magnitudes, photParams, gamma=shortGamma
)
snr, gg = calcSNR_m5(magnitudes, totalDict.values(), magnitudes, photParams)
def testSignalToNoise(self):
"""
Test that calcSNR_m5 and calcSNR_sed give similar results
"""
defaults = LSSTdefaults()
photParams = PhotometricParameters()
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
skySED = Sed()
skySED.readSED_flambda(os.path.join(lsst.utils.getPackageDir("throughputs"), "baseline", "darksky.dat"))
m5 = []
for filt in totalDict:
m5.append(calcM5(skySED, totalDict[filt], hardwareDict[filt], photParams, seeing=defaults.seeing(filt)))
sedDir = lsst.utils.getPackageDir("sims_sed_library")
sedDir = os.path.join(sedDir, "starSED", "kurucz")
fileNameList = os.listdir(sedDir)
numpy.random.seed(42)
offset = numpy.random.random_sample(len(fileNameList)) * 2.0
for ix, name in enumerate(fileNameList):
if ix > 100:
break
spectrum = Sed()
spectrum.readSED_flambda(os.path.join(sedDir, name))
ff = spectrum.calcFluxNorm(m5[2] - offset[ix], totalDict.values()[2])
spectrum.multiplyFluxNorm(ff)
magList = []
controlList = []
magList = []
for filt in totalDict:
controlList.append(
calcSNR_sed(
spectrum, totalDict[filt], skySED, hardwareDict[filt], photParams, defaults.seeing(filt)
)
)
magList.append(spectrum.calcMag(totalDict[filt]))
testList, gammaList = calcSNR_m5(
numpy.array(magList), numpy.array(totalDict.values()), numpy.array(m5), photParams
)
for tt, cc in zip(controlList, testList):
msg = "%e != %e " % (tt, cc)
self.assertTrue(numpy.abs(tt / cc - 1.0) < 0.001, msg=msg)
class TestSkyBrightness(unittest.TestCase):
totalbandpassdict, hwbandpassdict = BandpassDict.loadBandpassesFromFiles()
skycalc = SkyCalculations(photparams="LSST", hwBandpassDict=hwbandpassdict)
def test_skymags(self):
skymag = self.skycalc.skymag('g', 0.925184, -0.4789, 61044.077855)
assert_almost_equal(skymag, 18.8900, decimal=2)
def test_skyDepths(self):
m5 = self.skycalc.fiveSigmaDepth('g',
1.086662,
0.925184,
-0.4789,
61044.077855,
provided_airmass=1.008652,
use_provided_airmass=True)
assert_almost_equal(m5, 23.0601, decimal=2)
def __init__(self,
obs_metadata,
photParams,
seed=None,
bandpassDict=None,
addNoise=True,
addBackground=True,
logger=None):
"""
@param [in] addNoise is a boolean telling the wrapper whether or not
to add noise to the image
@param [in] addBackground is a boolean telling the wrapper whether
or not to add the skybackground to the image
@param [in] seed is an (optional) int that will seed the
random number generator used by the noise model. Defaults to None,
which causes GalSim to generate the seed from the system.
"""
self.obs_metadata = obs_metadata
self.photParams = photParams
if bandpassDict is None:
self.bandpassDict = BandpassDict.loadBandpassesFromFiles()[0]
# Computing the skybrightness.SkyModel object is expensive, so
# do it only once in the constructor.
self.skyModel = skybrightness.SkyModel(mags=False)
self.addNoise = addNoise
self.addBackground = addBackground
if logger is not None:
self.logger = logger
else:
self.logger = get_logger('INFO')
if seed is None:
self.randomNumbers = galsim.UniformDeviate()
else:
self.randomNumbers = galsim.UniformDeviate(seed)
def testUncertaintyExceptions(self):
"""
Test that calcSNR_m5 raises exceptions when it needs to
"""
totalDict, hardwareDict = BandpassDict.loadBandpassesFromFiles()
magnitudes = numpy.array([22.0, 23.0, 24.0, 25.0, 26.0, 27.0])
shortMagnitudes = numpy.array([22.0])
photParams = PhotometricParameters()
shortGamma = numpy.array([1.0, 1.0])
self.assertRaises(RuntimeError, calcSNR_m5, magnitudes,
totalDict.values(), shortMagnitudes, photParams)
self.assertRaises(RuntimeError, calcSNR_m5, shortMagnitudes,
totalDict.values(), magnitudes, photParams)
self.assertRaises(RuntimeError,
calcSNR_m5,
magnitudes,
totalDict.values(),
magnitudes,
photParams,
gamma=shortGamma)
snr, gg = calcSNR_m5(magnitudes, totalDict.values(), magnitudes,
photParams)
import os
import numpy as np
from lsst.utils import getPackageDir
from lsst.sims.catUtils.exampleCatalogDefinitions import DefaultPhoSimHeaderMap
from lsst.sims.catUtils.exampleCatalogDefinitions import write_phoSim_header
from lsst.sims.catUtils.utils import ObservationMetaDataGenerator
from lsst.sims.coordUtils import pupilCoordsFromFocalPlaneCoordsLSST
from lsst.sims.utils import raDecFromPupilCoords
from lsst.sims.utils import angularSeparation
from lsst.sims.utils import Site
from lsst.sims.photUtils import BandpassDict, Sed
from lsst.sims.utils import altAzPaFromRaDec
from lsst.sims.utils import ObservationMetaData
bp_dict, hw_dict = BandpassDict.loadBandpassesFromFiles()
opsimdb = os.path.join('/Users/danielsf/physics/lsst_150412',
'Development', 'garage', 'OpSimData',
'minion_1016_sqlite.db')
obs_gen = ObservationMetaDataGenerator(opsimdb)
obs_list = obs_gen.getObservationMetaData(moonAlt=(-90.0, -50.0),
altitude=(55.0, 57.0),
fieldDec=(-10.0, 10.0))
assert len(obs_list) > 0
obs_root = obs_list[0]
obs_root.site = Site(name='LSST', pressure=0.0, humidity=0.0)
phosim_header = DefaultPhoSimHeaderMap
def validate_instance_catalog_magnitudes(cat_dir, obsid, seed=99, nrows=-1):
"""
Parameters
----------
cat_dir is the parent dir of $obsid
obsid is the obsHistID of the pointing
seed is the seed for a random number generator
nrows is the number of galaxies to test (if <0, test all of them)
"""
agn_dtype = np.dtype([('galaxy_id', int), ('twinkles_id', int)])
agn_cache = np.genfromtxt(os.path.join(os.environ['TWINKLES_DIR'], 'data',
'cosmoDC2_v1.1.4_agn_cache.csv'),
dtype=agn_dtype,
delimiter=',',
skip_header=1)
sne_cache = np.genfromtxt(os.path.join(os.environ['TWINKLES_DIR'], 'data',
'cosmoDC2_v1.1.4_sne_cache.csv'),
dtype=agn_dtype,
delimiter=',',
skip_header=1)
sprinkled_gid = np.append(agn_cache['galaxy_id'], sne_cache['galaxy_id'])
colnames = [
'obj', 'uniqueID', 'ra', 'dec', 'magnorm', 'sed', 'redshift', 'g1',
'g2', 'kappa', 'dra', 'ddec', 'src_type', 'major', 'minor',
'positionAngle', 'sindex', 'dust_rest', 'rest_av', 'rest_rv',
'dust_obs', 'obs_av', 'obs_rv'
]
to_drop = [
'obj', 'g1', 'g2', 'kappa', 'dra', 'ddec', 'src_type', 'major',
'minor', 'positionAngle', 'sindex', 'dust_rest', 'dust_obs'
]
col_types = {
'magnorm': float,
'redshift': float,
'rest_av': float,
'rest_rv': float,
'sed': bytes,
'uniqueID': int
}
assert os.path.isdir(cat_dir)
data_dir = os.path.join(cat_dir, '%.8d' % obsid)
if not os.path.isdir(data_dir):
raise RuntimeError('\n\n%s\nis not a dir\n\n' % data_dir)
phosim_file = os.path.join(data_dir, 'phosim_cat_%d.txt' % obsid)
assert os.path.isfile(phosim_file)
bandpass_name = None
bandpass_name_list = 'ugrizy'
with open(phosim_file, 'r') as in_file:
for line in in_file:
params = line.strip().split()
if params[0] == 'filter':
bandpass_name = bandpass_name_list[int(params[1])]
assert bandpass_name is not None
(tot_dict, hw_dict) = BandpassDict.loadBandpassesFromFiles()
bandpass = hw_dict[bandpass_name]
disk_file = os.path.join(data_dir, 'disk_gal_cat_%d.txt.gz' % obsid)
if not os.path.isfile(disk_file):
raise RuntimeError("%s is not a file" % disk_file)
bulge_file = os.path.join(data_dir, 'bulge_gal_cat_%d.txt.gz' % obsid)
assert os.path.isfile(bulge_file)
knots_file = os.path.join(data_dir, 'knots_cat_%d.txt.gz' % obsid)
assert os.path.isfile(knots_file)
print('reading disks')
disk_df = pd.read_csv(disk_file,
delimiter=' ',
compression='gzip',
names=colnames,
dtype=col_types,
nrows=None)
disk_df.drop(labels=to_drop, axis='columns', inplace=True)
print('read disks')
disk_df['galaxy_id'] = pd.Series(disk_df['uniqueID'] // 1024,
index=disk_df.index)
disk_df = disk_df.set_index('galaxy_id')
print('reading bulges')
bulge_df = pd.read_csv(bulge_file,
delimiter=' ',
compression='gzip',
names=colnames,
dtype=col_types,
nrows=None)
bulge_df.drop(labels=to_drop, axis='columns', inplace=True)
print('read bulges')
bulge_df['galaxy_id'] = pd.Series(bulge_df['uniqueID'] // 1024,
index=bulge_df.index)
bulge_df = bulge_df.set_index('galaxy_id')
for ii in range(len(colnames)):
colnames[ii] = colnames[ii] + '_knots'
for ii in range(len(to_drop)):
to_drop[ii] = to_drop[ii] + '_knots'
print('reading knots')
knots_df = pd.read_csv(knots_file,
delimiter=' ',
compression='gzip',
names=colnames,
dtype=col_types,
nrows=None)
knots_df.drop(labels=to_drop, axis='columns', inplace=True)
print('read knots')
knots_df['galaxy_id'] = pd.Series(knots_df['uniqueID_knots'] // 1024,
index=knots_df.index)
knots_df = knots_df.set_index('galaxy_id')
wanted_col = [
'sed', 'magnorm', 'redshift', 'rest_av', 'rest_rv', 'ra', 'dec'
]
galaxy_df = disk_df[wanted_col].join(bulge_df[wanted_col],
how='outer',
lsuffix='_disk',
rsuffix='_bulge')
for ii in range(len(wanted_col)):
wanted_col[ii] = wanted_col[ii] + '_knots'
galaxy_df = galaxy_df.join(knots_df[wanted_col],
how='outer',
rsuffix='_knots')
valid_galaxies = np.where(
np.logical_not(np.in1d(galaxy_df.index, sprinkled_gid)))
galaxy_df = galaxy_df.iloc[valid_galaxies]
ra_center = np.nanmedian(galaxy_df['ra_disk'].values)
dec_center = np.nanmedian(galaxy_df['dec_disk'].values)
dd = angularSeparation(ra_center, dec_center, galaxy_df['ra_disk'].values,
galaxy_df['dec_disk'].values)
radius_deg = np.nanmax(dd)
ra_rad = np.radians(ra_center)
dec_rad = np.radians(dec_center)
vv = np.array([
np.cos(ra_rad) * np.cos(dec_rad),
np.sin(ra_rad) * np.cos(dec_rad),
np.sin(dec_rad)
])
healpix_list = healpy.query_disc(32,
vv,
np.radians(radius_deg),
nest=False,
inclusive=True)
gal_id_values = galaxy_df.index.values
cat = GCRCatalogs.load_catalog('cosmoDC2_v1.1.4_image')
cat_qties = {}
cat_qties['galaxy_id'] = []
cat_qties['ra'] = []
cat_qties['dec'] = []
for hp in healpix_list:
hp_query = GCRQuery('healpix_pixel==%d' % hp)
local_qties = cat.get_quantities(['galaxy_id', 'ra', 'dec'],
native_filters=[hp_query])
valid = np.in1d(local_qties['galaxy_id'], gal_id_values)
if valid.any():
for k in local_qties:
cat_qties[k].append(local_qties[k][valid])
for k in cat_qties:
cat_qties[k] = np.concatenate(cat_qties[k])
cat_dexes = np.arange(len(cat_qties['galaxy_id']), dtype=int)
if nrows > 0:
rng = np.random.RandomState(seed)
dexes = rng.choice(galaxy_df.index.values, size=nrows, replace=False)
galaxy_df = galaxy_df.loc[dexes]
galaxy_df = galaxy_df.sort_index()
invalid_knots = np.where(
np.logical_not(
np.isfinite(galaxy_df['magnorm_knots'].values.astype(np.float))))
dd = angularSeparation(ra_center, dec_center, cat_qties['ra'],
cat_qties['dec'])
dd_cut = np.where(dd < (radius_deg + 0.05))
gid = cat_qties['galaxy_id'][dd_cut]
cat_dexes = cat_dexes[dd_cut]
in1d_valid_dexes = np.where(
np.in1d(gid, galaxy_df.index.values, assume_unique=True))
valid_dexes = cat_dexes[in1d_valid_dexes]
gid = gid[in1d_valid_dexes]
sorted_dex = np.argsort(gid)
valid_dexes = valid_dexes[sorted_dex]
assert len(gid) == len(galaxy_df.index.values)
np.testing.assert_array_equal(gid[sorted_dex], galaxy_df.index.values)
mag_name = 'mag_true_%s_lsst' % bandpass_name
qties = {}
qties['galaxy_id'] = []
qties[mag_name] = []
for hp in healpix_list:
hp_query = GCRQuery('healpix_pixel==%d' % hp)
local_qties = cat.get_quantities(['galaxy_id', mag_name],
native_filters=[hp_query])
valid = np.in1d(local_qties['galaxy_id'], gal_id_values)
if valid.any():
for k in local_qties:
qties[k].append(local_qties[k][valid])
for k in qties:
qties[k] = np.concatenate(qties[k])
np.testing.assert_array_equal(qties['galaxy_id'], cat_qties['galaxy_id'])
mags = qties[mag_name][valid_dexes]
gid = qties['galaxy_id'][valid_dexes]
assert len(gid) == len(mags)
assert len(mags) > 0
if nrows > 0:
assert len(mags) == nrows
t_start = time.time()
n_proc = 3
d_proc = len(gid) // n_proc
mgr = multiprocessing.Manager()
out_dict = mgr.dict()
p_list = []
for i_start in range(0, len(gid), d_proc):
mag_true = mags[i_start:i_start + d_proc]
galaxy_arr = galaxy_df.iloc[i_start:i_start + d_proc]
p = multiprocessing.Process(target=validate_batch,
args=(mag_true, galaxy_arr, bandpass,
out_dict))
p.start()
p_list.append(p)
for p in p_list:
p.join()
assert len(list(out_dict.keys())) > 0
d_mag_max = 0.0
for k in out_dict.keys():
if out_dict[k] > d_mag_max:
d_mag_max = out_dict[k]
if d_mag_max > 1.0e-5:
raise RuntimeError("\nobsHistID failed magnitud validation\n"
"d_mag_max %e" % d_mag_max)
def sed_from_galacticus_mags(galacticus_mags, redshift, redshift_true, H0, Om0,
wav_min, wav_width, obs_lsst_mags):
"""
Fit SEDs from sims_sed_library to Galacticus galaxies based on the
magnitudes in tophat filters.
Parameters
----------
galacticus_mags is a numpy array such that
galacticus_mags[i][j] is the magnitude of the jth star in the ith bandpass,
where the bandpasses are ordered in ascending order of minimum wavelength.
redshift is an array of redshifts for the galaxies being fit
(includes cosmology and proper motion)
redshift_true is an array of cosmological redshifts for the galaxies
being fit
H0 is the Hubbleparameter in units of km/s/Mpc
Om0 is the critical density parameter for matter
wav_min is a numpy array of the minimum wavelengths of the tophat
filters (in nm)
wav_grid is a numpy array of the widths of the tophat filters
(in nm)
ob_lsst_mags is a numpy array of observer frame LSST magnitudes.
obs_lsst_mags[0] will contain the u band magnitudes of every object.
Returns
-------
a numpy array of SED names and a numpy array of magNorms.
"""
if (not hasattr(sed_from_galacticus_mags, '_color_tree')
or not np.allclose(wav_min,
sed_from_galacticus_mags._wav_min,
atol=1.0e-10,
rtol=0.0)
or not np.allclose(wav_width,
sed_from_galacticus_mags._wav_width,
atol=1.0e-10,
rtol=0.0)):
(sed_names, sed_mag_list, sed_mag_norm, av_grid,
rv_grid) = _create_sed_library_mags(wav_min, wav_width)
assert rv_grid.min() > 0.0
assert len(np.where(np.logical_not(np.isfinite(rv_grid)))[0]) == 0
sed_colors = sed_mag_list[:, 1:] - sed_mag_list[:, :-1]
sed_from_galacticus_mags._sed_names = sed_names
sed_from_galacticus_mags._mag_norm = sed_mag_norm # N_sed
sed_from_galacticus_mags._av_grid = av_grid
sed_from_galacticus_mags._rv_grid = rv_grid
sed_from_galacticus_mags._sed_mags = sed_mag_list # N_sed by N_mag
sed_from_galacticus_mags._color_tree = scipy_spatial.cKDTree(
sed_colors)
sed_from_galacticus_mags._wav_min = wav_min
sed_from_galacticus_mags._wav_width = wav_width
if (not hasattr(sed_from_galacticus_mags, '_cosmo')
or np.abs(sed_from_galacticus_mags._cosmo.H() - H0) > 1.0e-6
or np.abs(sed_from_galacticus_mags._cosmo.OmegaMatter() - Om0) >
1.0e-6):
sed_from_galacticus_mags._cosmo = CosmologyObject(H0=H0, Om0=Om0)
galacticus_mags_t = np.asarray(galacticus_mags).T # N_star by N_mag
assert galacticus_mags_t.shape == (
len(redshift), sed_from_galacticus_mags._sed_mags.shape[1])
with np.errstate(invalid='ignore', divide='ignore'):
galacticus_colors = galacticus_mags_t[:,
1:] - galacticus_mags_t[:, :
-1] # N_star by (N_mag - 1)
t_start = time.time()
(sed_dist,
sed_idx) = sed_from_galacticus_mags._color_tree.query(galacticus_colors,
k=1)
# cKDTree returns an invalid index (==len(tree_data)) in cases
# where the distance is not finite
sed_idx = np.where(sed_idx < len(sed_from_galacticus_mags._sed_names),
sed_idx, 0)
distance_modulus = sed_from_galacticus_mags._cosmo.distanceModulus(
redshift=redshift_true)
output_names = sed_from_galacticus_mags._sed_names[sed_idx]
(lsst_bp_dict, dummy_bp_dict) = BandpassDict.loadBandpassesFromFiles()
output_mag_norm = np.zeros((6, len(output_names)), dtype=float)
base_norm = sed_from_galacticus_mags._mag_norm[sed_idx]
assert len(np.where(np.logical_not(np.isfinite(base_norm)))[0]) == 0
ccm_w = None
av_arr = sed_from_galacticus_mags._av_grid[sed_idx]
rv_arr = sed_from_galacticus_mags._rv_grid[sed_idx]
assert rv_arr.min() > 0.0
assert len(np.where(np.logical_not(np.isfinite(rv_arr)))[0]) == 0
for i_bp in range(6):
output_mag_norm[i_bp, :] = base_norm + distance_modulus
sed_dir = getPackageDir('sims_sed_library')
for i_obj in range(len(output_names)):
spec = Sed()
spec.readSED_flambda(os.path.join(sed_dir, output_names[i_obj]))
if ccm_w is None or not np.array_equal(spec.wavelen, ccm_w):
ccm_w = np.copy(spec.wavelen)
ax, bx = spec.setupCCM_ab()
spec.addDust(ax, bx, A_v=av_arr[i_obj], R_v=rv_arr[i_obj])
spec.redshiftSED(redshift[i_obj], dimming=True)
lsst_mags = lsst_bp_dict.magListForSed(spec)
d_mag = obs_lsst_mags[:, i_obj] - lsst_mags
output_mag_norm[:, i_obj] += d_mag
return (output_names, output_mag_norm, av_arr, rv_arr)
# Bandpasses and LSST related stuff
def atmTransName(airmass):
"""
obtain the filename with the transmissions for files for an airmass value closest to the requested value
"""
l = np.arange(1.0, 2.51, 0.1)
idx = np.abs(l - airmass).argmin()
a = np.int(10 * l[idx])
baseline = getPackageDir('THROUGHPUTS')
fname = os.path.join(baseline, 'atmos', 'atmos_{}_aerosol.dat'.format(a))
return fname
logger.info('Get Bandpasses')
totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles()
photparams = PhotometricParameters()
# Split the entries in the opsim database for parallelization
splits = 40
dfs = np.array_split(df, splits)
print(
'splitting dataframe of size {0} into {1} splits each of size {2}'.format(
len(df), splits, len(dfs[0])))
calcdfs = []
def recalcmags(j, lst=calcdfs):
logfname = 'newres_{}.log'.format(j)
with open(logfname, 'w') as f:
print('numpy version: ', np.__version__)
print('healpy dir', getPackageDir('healpy'))
print('sims_skybrightness_dir', getPackageDir('sims_skybrightness'))
print('sims_skybrightness_data_dir', getPackageDir('sims_skybrightness_data'))
print('obscond version', obscond.__version__)
logger.info('Start reading opsim database')
minion_out = '/local/lsst/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
#minion_out = '/Users/rbiswas/data/LSST/OpSimData/minion_1016_sqlite.db'
opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="unique_all")
# opsout = OpSimOutput.fromOpSimDB(minion_out, zeroDDFDithers=True, subset="ddf")
print('reading done\n')
df = opsout.summary.copy()
logger.info('Finished reading database at {}'.format(time.time()))
totalbpdict, hwbpdict = BandpassDict.loadBandpassesFromFiles()
photparams = PhotometricParameters()
# Split the entries in the opsim database for parallelization
splits = 1000
dfs = np.array_split(df, splits)
print('splitting dataframe of size {0} into {1} splits each of size {2}\n'.format(len(df), splits, len(dfs[0])))
calcdfs = []
def recalcmags(j):
logfname = 'newres_{}.log'.format(j)
tsplitstart = time.time()
with open(logfname, 'w') as f:
f.write('starting split {0} at time {1}\n'.format(j, tsplitstart))
df = dfs[j]
def _initializeGalSimInterpreter(self):
"""
This method creates the GalSimInterpreter (if it is None)
This method reads in all of the data about the camera and pass it into
the GalSimInterpreter.
This method calls _getBandpasses to construct the paths to
the files containing the bandpass data.
"""
if not isinstance(self.camera_wrapper, GalSimCameraWrapper):
raise RuntimeError("GalSimCatalog.camera_wrapper must be an instantiation of "
"GalSimCameraWrapper or one of its daughter classes\n"
"It is actually of type %s" % str(type(self.camera_wrapper)))
if self.galSimInterpreter is None:
# This list will contain instantiations of the GalSimDetector class
# (see galSimInterpreter.py), which stores detector information in a way
# that the GalSimInterpreter will understand
detectors = []
for dd in self.camera_wrapper.camera:
if dd.getType() == WAVEFRONT or dd.getType() == GUIDER:
# This package does not yet handle the 90-degree rotation
# in WCS that occurs for wavefront or guide sensors
continue
if self.allowed_chips is None or dd.getName() in self.allowed_chips:
centerPupil = self.camera_wrapper.getCenterPupil(dd.getName())
centerPixel = self.camera_wrapper.getCenterPixel(dd.getName())
translationPupil = self.camera_wrapper.pupilCoordsFromPixelCoords(centerPixel.getX()+1,
centerPixel.getY()+1,
dd.getName())
plateScale = np.sqrt(np.power(translationPupil[0]-centerPupil.getX(), 2) +
np.power(translationPupil[1]-centerPupil.getY(), 2))/np.sqrt(2.0)
plateScale = 3600.0*np.degrees(plateScale)
# make a detector-custom photParams that copies all of the quantities
# in the catalog photParams, except the platescale, which is
# calculated above
params = PhotometricParameters(exptime=self.photParams.exptime,
nexp=self.photParams.nexp,
effarea=self.photParams.effarea,
gain=self.photParams.gain,
readnoise=self.photParams.readnoise,
darkcurrent=self.photParams.darkcurrent,
othernoise=self.photParams.othernoise,
platescale=plateScale)
detector = GalSimDetector(dd.getName(), self.camera_wrapper,
obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch,
photParams=params)
detectors.append(detector)
if not hasattr(self, 'bandpassDict'):
if self.noise_and_background is not None:
if self.obs_metadata.m5 is None:
raise RuntimeError('WARNING in GalSimCatalog; you did not specify m5 in your '
'obs_metadata. m5 is required in order to '
'add noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.m5:
raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' +
'm5 values for all of your bandpasses \n' +
'bandpass has: %s \n' % self.bandpassNames.__repr__() +
'm5 has: %s ' % list(self.obs_metadata.m5.keys()).__repr__())
if self.obs_metadata.seeing is None:
raise RuntimeError('WARNING in GalSimCatalog; you did not specify seeing in your '
'obs_metadata. seeing is required in order to add '
'noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.seeing:
raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' +
'seeing values for all of your bandpasses \n' +
'bandpass has: %s \n' % self.bandpassNames.__repr__() +
'seeing has: %s ' % list(self.obs_metadata.seeing.keys()).__repr__())
(self.bandpassDict,
hardwareDict) = BandpassDict.loadBandpassesFromFiles(bandpassNames=self.bandpassNames,
filedir=self.bandpassDir,
bandpassRoot=self.bandpassRoot,
componentList=self.componentList,
atmoTransmission=os.path.join(self.bandpassDir,
self.atmoTransmissionName))
self.galSimInterpreter = GalSimInterpreter(obs_metadata=self.obs_metadata,
epoch=self.db_obj.epoch,
detectors=detectors,
bandpassDict=self.bandpassDict,
noiseWrapper=self.noise_and_background,
seed=self.seed)
self.galSimInterpreter.setPSF(PSF=self.PSF)
def _initializeGalSimInterpreter(self):
"""
This method creates the GalSimInterpreter (if it is None)
This method reads in all of the data about the camera and pass it into
the GalSimInterpreter.
This method calls _getBandpasses to construct the paths to
the files containing the bandpass data.
"""
if not isinstance(self.camera_wrapper, GalSimCameraWrapper):
raise RuntimeError(
"GalSimCatalog.camera_wrapper must be an instantiation of "
"GalSimCameraWrapper or one of its daughter classes\n"
"It is actually of type %s" % str(type(self.camera_wrapper)))
if self.galSimInterpreter is None:
# This list will contain instantiations of the GalSimDetector class
# (see galSimInterpreter.py), which stores detector information in a way
# that the GalSimInterpreter will understand
detectors = []
for dd in self.camera_wrapper.camera:
if dd.getType() == WAVEFRONT or dd.getType() == GUIDER:
# This package does not yet handle the 90-degree rotation
# in WCS that occurs for wavefront or guide sensors
continue
if self.allowed_chips is None or dd.getName(
) in self.allowed_chips:
detectors.append(
make_galsim_detector(self.camera_wrapper,
dd.getName(),
self.photParams,
self.obs_metadata,
epoch=self.db_obj.epoch))
if not hasattr(self, 'bandpassDict'):
if self.noise_and_background is not None:
if self.obs_metadata.m5 is None:
raise RuntimeError(
'WARNING in GalSimCatalog; you did not specify m5 in your '
'obs_metadata. m5 is required in order to '
'add noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.m5:
raise RuntimeError(
'WARNING in GalSimCatalog; your obs_metadata does not have '
+ 'm5 values for all of your bandpasses \n' +
'bandpass has: %s \n' %
self.bandpassNames.__repr__() + 'm5 has: %s ' %
list(self.obs_metadata.m5.keys()).__repr__())
if self.obs_metadata.seeing is None:
raise RuntimeError(
'WARNING in GalSimCatalog; you did not specify seeing in your '
'obs_metadata. seeing is required in order to add '
'noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.seeing:
raise RuntimeError(
'WARNING in GalSimCatalog; your obs_metadata does not have '
+
'seeing values for all of your bandpasses \n' +
'bandpass has: %s \n' %
self.bandpassNames.__repr__() +
'seeing has: %s ' %
list(self.obs_metadata.seeing.keys()).__repr__(
))
(self.bandpassDict,
hardwareDict) = BandpassDict.loadBandpassesFromFiles(
bandpassNames=self.bandpassNames,
filedir=self.bandpassDir,
bandpassRoot=self.bandpassRoot,
componentList=self.componentList,
atmoTransmission=os.path.join(self.bandpassDir,
self.atmoTransmissionName))
self.galSimInterpreter = GalSimInterpreter(
obs_metadata=self.obs_metadata,
epoch=self.db_obj.epoch,
detectors=detectors,
bandpassDict=self.bandpassDict,
noiseWrapper=self.noise_and_background,
seed=self.seed)
self.galSimInterpreter.setPSF(PSF=self.PSF)
t_start = time.time()
sed_name_list, mag_norm_list = sed_from_galacticus_mags(disk_mags, true_redshift_list)
rest_sed_name_list, rest_mag_norm_list=sed_from_galacticus_mags(disk_mags, np.zeros(len(true_redshift_list), dtype=float))
np.testing.assert_array_equal(sed_name_list, rest_sed_name_list)
print("fitting %d took %.3e" % (len(sed_name_list), time.time()-t_start))
print("mag norm %e %e %e" % (mag_norm_list.min(), np.median(mag_norm_list), mag_norm_list.max()))
assert len(sed_name_list) == len(first_disk)
sed_dir = getPackageDir('sims_sed_library')
gal_sed_dir = os.path.join(sed_dir, 'galaxySED')
from lsst.sims.photUtils import Sed, BandpassDict, getImsimFluxNorm
total_bp_dict, lsst_bp_dict = BandpassDict.loadBandpassesFromFiles()
worst_dist = -1.0
av_valid = np.where(np.logical_and(av_list>0.01, np.logical_and(ebv_list>0.0, av_list<5.0)))
sed_name_list = sed_name_list[av_valid]
mag_norm_list = mag_norm_list[av_valid]
rest_mag_norm_list = rest_mag_norm_list[av_valid]
true_redshift_list = true_redshift_list[av_valid]
full_redshift_list = full_redshift_list[av_valid]
av_list = av_list[av_valid]
ebv_list = ebv_list[av_valid]
u_control = u_control[av_valid]
g_control = g_control[av_valid]
r_control = r_control[av_valid]
def _initializeGalSimInterpreter(self):
"""
This method creates the GalSimInterpreter (if it is None)
This method reads in all of the data about the camera and pass it into
the GalSimInterpreter.
This method calls _getBandpasses to construct the paths to
the files containing the bandpass data.
"""
if self.galSimInterpreter is None:
# This list will contain instantiations of the GalSimDetector class
# (see galSimInterpreter.py), which stores detector information in a way
# that the GalSimInterpreter will understand
detectors = []
for dd in self.camera:
if self.allowed_chips is None or dd.getName(
) in self.allowed_chips:
cs = dd.makeCameraSys(PUPIL)
centerPupil = self.camera.transform(
dd.getCenter(FOCAL_PLANE), cs).getPoint()
centerPixel = dd.getCenter(PIXELS).getPoint()
translationPixel = afwGeom.Point2D(centerPixel.getX() + 1,
centerPixel.getY() + 1)
translationPupil = self.camera.transform(
dd.makeCameraPoint(translationPixel, PIXELS),
cs).getPoint()
plateScale = np.sqrt(
np.power(translationPupil.getX() -
centerPupil.getX(), 2) +
np.power(translationPupil.getY() -
centerPupil.getY(), 2)) / np.sqrt(2.0)
plateScale = 3600.0 * np.degrees(plateScale)
# make a detector-custom photParams that copies all of the quantities
# in the catalog photParams, except the platescale, which is
# calculated above
params = PhotometricParameters(
exptime=self.photParams.exptime,
nexp=self.photParams.nexp,
effarea=self.photParams.effarea,
gain=self.photParams.gain,
readnoise=self.photParams.readnoise,
darkcurrent=self.photParams.darkcurrent,
othernoise=self.photParams.othernoise,
platescale=plateScale)
detector = GalSimDetector(dd,
self.camera,
obs_metadata=self.obs_metadata,
epoch=self.db_obj.epoch,
photParams=params)
detectors.append(detector)
if not hasattr(self, 'bandpassDict'):
if self.noise_and_background is not None:
if self.obs_metadata.m5 is None:
raise RuntimeError(
'WARNING in GalSimCatalog; you did not specify m5 in your '
'obs_metadata. m5 is required in order to '
'add noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.m5:
raise RuntimeError(
'WARNING in GalSimCatalog; your obs_metadata does not have '
+ 'm5 values for all of your bandpasses \n' +
'bandpass has: %s \n' %
self.bandpassNames.__repr__() + 'm5 has: %s ' %
list(self.obs_metadata.m5.keys()).__repr__())
if self.obs_metadata.seeing is None:
raise RuntimeError(
'WARNING in GalSimCatalog; you did not specify seeing in your '
'obs_metadata. seeing is required in order to add '
'noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.seeing:
raise RuntimeError(
'WARNING in GalSimCatalog; your obs_metadata does not have '
+
'seeing values for all of your bandpasses \n' +
'bandpass has: %s \n' %
self.bandpassNames.__repr__() +
'seeing has: %s ' %
list(self.obs_metadata.seeing.keys()).__repr__(
))
(self.bandpassDict,
hardwareDict) = BandpassDict.loadBandpassesFromFiles(
bandpassNames=self.bandpassNames,
filedir=self.bandpassDir,
bandpassRoot=self.bandpassRoot,
componentList=self.componentList,
atmoTransmission=os.path.join(self.bandpassDir,
self.atmoTransmissionName))
self.galSimInterpreter = GalSimInterpreter(
obs_metadata=self.obs_metadata,
epoch=self.db_obj.epoch,
detectors=detectors,
bandpassDict=self.bandpassDict,
noiseWrapper=self.noise_and_background,
seed=self.seed)
self.galSimInterpreter.setPSF(PSF=self.PSF)
def _initializeGalSimInterpreter(self):
"""
This method creates the GalSimInterpreter (if it is None)
This method reads in all of the data about the camera and pass it into
the GalSimInterpreter.
This method calls _getBandpasses to construct the paths to
the files containing the bandpass data.
"""
if self.galSimInterpreter is None:
#This list will contain instantiations of the GalSimDetector class
#(see galSimInterpreter.py), which stores detector information in a way
#that the GalSimInterpreter will understand
detectors = []
for dd in self.camera:
if self.allowed_chips is None or dd.getName() in self.allowed_chips:
cs = dd.makeCameraSys(PUPIL)
centerPupil = self.camera.transform(dd.getCenter(FOCAL_PLANE),cs).getPoint()
centerPixel = dd.getCenter(PIXELS).getPoint()
translationPixel = afwGeom.Point2D(centerPixel.getX()+1, centerPixel.getY()+1)
translationPupil = self.camera.transform(
dd.makeCameraPoint(translationPixel, PIXELS), cs).getPoint()
plateScale = numpy.sqrt(numpy.power(translationPupil.getX()-centerPupil.getX(),2)+
numpy.power(translationPupil.getY()-centerPupil.getY(),2))/numpy.sqrt(2.0)
plateScale = 3600.0*numpy.degrees(plateScale)
#make a detector-custom photParams that copies all of the quantities
#in the catalog photParams, except the platescale, which is
#calculated above
params = PhotometricParameters(exptime=self.photParams.exptime,
nexp=self.photParams.nexp,
effarea=self.photParams.effarea,
gain=self.photParams.gain,
readnoise=self.photParams.readnoise,
darkcurrent=self.photParams.darkcurrent,
othernoise=self.photParams.othernoise,
platescale=plateScale)
detector = GalSimDetector(dd, self.camera,
obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch,
photParams=params)
detectors.append(detector)
if not hasattr(self, 'bandpassDict'):
if self.noise_and_background is not None:
if self.obs_metadata.m5 is None:
raise RuntimeError('WARNING in GalSimCatalog; you did not specify m5 in your '+
'obs_metadata. m5 is required in order to add noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.m5:
raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' +
'm5 values for all of your bandpasses \n' +
'bandpass has: %s \n' % self.bandpassNames.__repr__() +
'm5 has: %s ' % self.obs_metadata.m5.keys().__repr__())
if self.obs_metadata.seeing is None:
raise RuntimeError('WARNING in GalSimCatalog; you did not specify seeing in your '+
'obs_metadata. seeing is required in order to add noise to your images')
for name in self.bandpassNames:
if name not in self.obs_metadata.seeing:
raise RuntimeError('WARNING in GalSimCatalog; your obs_metadata does not have ' +
'seeing values for all of your bandpasses \n' +
'bandpass has: %s \n' % self.bandpassNames.__repr__() +
'seeing has: %s ' % self.obs_metadata.seeing.keys().__repr__())
self.bandpassDict, hardwareDict = BandpassDict.loadBandpassesFromFiles(bandpassNames=self.bandpassNames,
filedir=self.bandpassDir,
bandpassRoot=self.bandpassRoot,
componentList=self.componentList,
atmoTransmission=os.path.join(self.bandpassDir, self.atmoTransmissionName))
self.galSimInterpreter = GalSimInterpreter(obs_metadata=self.obs_metadata, epoch=self.db_obj.epoch, detectors=detectors,
bandpassDict=self.bandpassDict, noiseWrapper=self.noise_and_background,
seed=self.seed)
self.galSimInterpreter.setPSF(PSF=self.PSF)
def __init__(self):
self.lsst_bandpasses = BandpassDict.loadBandpassesFromFiles()[0]
self._gs_bandpasses = dict()
