def testEBV(self):
ebvObject = EBVbase()
ra = []
dec = []
gLat = []
gLon = []
for i in range(10):
ra.append(i*2.0*np.pi/10.0)
dec.append(i*np.pi/10.0)
gLat.append(-0.5*np.pi+i*np.pi/10.0)
gLon.append(i*2.0*np.pi/10.0)
equatorialCoordinates = np.array([ra, dec])
galacticCoordinates = np.array([gLon, gLat])
ebvOutput = ebvObject.calculateEbv(equatorialCoordinates=equatorialCoordinates)
self.assertEqual(len(ebvOutput), len(ra))
ebvOutput = ebvObject.calculateEbv(galacticCoordinates=galacticCoordinates)
self.assertEqual(len(ebvOutput), len(gLon))
self.assertGreater(len(ebvOutput), 0)
self.assertRaises(RuntimeError, ebvObject.calculateEbv, equatorialCoordinates=equatorialCoordinates,
galacticCoordinates=galacticCoordinates)
self.assertRaises(RuntimeError, ebvObject.calculateEbv,
equatorialCoordinates=None, galacticCoordinates=None)
self.assertRaises(RuntimeError, ebvObject.calculateEbv)
def testEBV(self):
ebvObject = EBVbase()
ra = []
dec = []
gLat = []
gLon = []
for i in range(10):
ra.append(i * 2.0 * np.pi / 10.0)
dec.append(i * np.pi / 10.0)
gLat.append(-0.5 * np.pi + i * np.pi / 10.0)
gLon.append(i * 2.0 * np.pi / 10.0)
equatorialCoordinates = np.array([ra, dec])
galacticCoordinates = np.array([gLon, gLat])
ebvOutput = ebvObject.calculateEbv(
equatorialCoordinates=equatorialCoordinates)
self.assertEqual(len(ebvOutput), len(ra))
ebvOutput = ebvObject.calculateEbv(
galacticCoordinates=galacticCoordinates)
self.assertEqual(len(ebvOutput), len(gLon))
self.assertGreater(len(ebvOutput), 0)
self.assertRaises(RuntimeError,
ebvObject.calculateEbv,
equatorialCoordinates=equatorialCoordinates,
galacticCoordinates=galacticCoordinates)
self.assertRaises(RuntimeError,
ebvObject.calculateEbv,
equatorialCoordinates=None,
galacticCoordinates=None)
self.assertRaises(RuntimeError, ebvObject.calculateEbv)
def test_cache(self):
"""
Test that EBVbase() only loads each dust map once
"""
sims_clean_up()
self.assertEqual(len(EBVbase._ebv_map_cache), 0)
ebv1 = EBVbase()
ebv1.load_ebvMapNorth()
ebv1.load_ebvMapSouth()
self.assertEqual(len(EBVbase._ebv_map_cache), 2)
ebv2 = EBVbase()
ebv2.load_ebvMapNorth()
ebv2.load_ebvMapSouth()
self.assertEqual(len(EBVbase._ebv_map_cache), 2)
rng = np.random.RandomState(881)
ra_list = rng.random_sample(10)*2.0*np.pi
dec_list = rng.random_sample(10)*np.pi - 0.5*np.pi
ebv1_vals = ebv1.calculateEbv(equatorialCoordinates=np.array([ra_list, dec_list]))
ebv2_vals = ebv2.calculateEbv(equatorialCoordinates=np.array([ra_list, dec_list]))
self.assertEqual(len(EBVbase._ebv_map_cache), 2)
np.testing.assert_array_equal(ebv1_vals, ebv2_vals)
def test_cache(self):
"""
Test that EBVbase() only loads each dust map once
"""
sims_clean_up()
self.assertEqual(len(EBVbase._ebv_map_cache), 0)
ebv1 = EBVbase()
ebv1.load_ebvMapNorth()
ebv1.load_ebvMapSouth()
self.assertEqual(len(EBVbase._ebv_map_cache), 2)
ebv2 = EBVbase()
ebv2.load_ebvMapNorth()
ebv2.load_ebvMapSouth()
self.assertEqual(len(EBVbase._ebv_map_cache), 2)
rng = np.random.RandomState(881)
ra_list = rng.random_sample(10) * 2.0 * np.pi
dec_list = rng.random_sample(10) * np.pi - 0.5 * np.pi
ebv1_vals = ebv1.calculateEbv(
equatorialCoordinates=np.array([ra_list, dec_list]))
ebv2_vals = ebv2.calculateEbv(
equatorialCoordinates=np.array([ra_list, dec_list]))
self.assertEqual(len(EBVbase._ebv_map_cache), 2)
np.testing.assert_array_equal(ebv1_vals, ebv2_vals)
class SNObject(sncosmo.Model):
"""
Extension of the SNCosmo `TimeSeriesModel` to include more parameters and
use methods in the catsim stack. We constrain ourselves to the use of a
specific SALT model for the Supernova (Salt2-Extended), and set its MW
extinction to be 0, since we will use the LSST software to calculate
extinction.
Parameters
----------
ra : float
ra of the SN in degrees
dec : float
dec of the SN in degrees
Attributes
----------
_ra : float or None
ra of the SN in radians
_dec : float or None
dec of the SN in radians
skycoord : `np.ndarray' of size 2 or None
np.array([[ra], [dec]]), which are in radians
ebvofMW : float or None
mwebv value calculated from the self.skycoord if not None, or set to
a value using self.set_MWebv. If neither of these are done, this value
will be None, leading to exceptions in extinction calculation.
Therefore, the value must be set explicitly to 0. to get unextincted
quantities.
rectifySED : Bool, True by Default
if the SED is negative at the requested time and wavelength, return 0.
instead of the negative value.
Methods
-------
Examples
--------
>>> SNObject = SNObject(ra=30., dec=60.)
>>> SNObject._ra
>>> 0.5235987755982988
>>> SNObject._dec
>>> 1.0471975511965976
"""
def __init__(self, ra=None, dec=None, source='salt2-extended'):
"""
Instantiate object
Parameters
----------
ra : float
ra of the SN in degrees
dec : float
dec of the SN in degrees
source : instance of `sncosmo.SALT2Source`, optional, defaults to using salt2-extended
source class to define the model
"""
dust = sncosmo.CCM89Dust()
sncosmo.Model.__init__(self, source=source, effects=[dust, dust],
effect_names=['host', 'mw'],
effect_frames=['rest', 'obs'])
# Current implementation of Model has a default value of mwebv = 0.
# ie. no extinction, but this is not part of the API, so should not
# depend on it, set explicitly in order to unextincted SED from
# SNCosmo. We will use catsim extinction from `lsst.sims.photUtils`.
self.ModelSource = source
self.set(mwebv=0.)
# self._ra, self._dec is initialized as None for cases where ra, dec
# is not provided
self._ra = None
self._dec = None
# ra, dec is input in degrees
# If provided, set _ra, _dec in radians
self._hascoords = True
if dec is None:
self._hascoords = False
if ra is None:
self._hascoords = False
# Satisfied that coordinates provided are floats
if self._hascoords:
self.setCoords(ra, dec)
# For values of ra, dec, the E(B-V) is calculated directly
# from DustMaps
self.lsstmwebv = EBVbase()
self.ebvofMW = None
if self._hascoords:
self.mwEBVfromMaps()
# Behavior of model outside temporal range :
# if 'zero' then all fluxes outside the temporal range of the model
# are set to 0.
self._modelOutSideTemporalRange = 'zero'
# SED will be rectified to 0. for negative values of SED if this
# attribute is set to True
self.rectifySED = True
return
@property
def SNstate(self):
"""
Dictionary summarizing the state of SNObject. Can be used to
serialize to disk, and create SNObject from SNstate
Returns : Dictionary with values of parameters of the model.
"""
statedict = dict()
# SNCosmo Parameters
statedict['ModelSource'] = self.source.name
for param_name in self.param_names:
statedict[param_name] = self.get(param_name)
# New Attributes
# statedict['lsstmwebv'] = self.lsstmwebv
statedict['_ra'] = self._ra
statedict['_dec'] = self._dec
statedict['MWE(B-V)'] = self.ebvofMW
return statedict
@classmethod
def fromSNState(cls, snState):
"""
creates an instance of SNObject with a state described by snstate.
Parameters
----------
snState: Dictionary summarizing the state of SNObject
Returns
-------
Instance of SNObject class with attributes set by snstate
Example
-------
"""
# Separate into SNCosmo parameters and SNObject parameters
dust = sncosmo.CCM89Dust()
sncosmoModel = sncosmo.Model(source=snState['ModelSource'],
effects=[dust, dust],
effect_names=['host', 'mw'],
effect_frames=['rest', 'obs'])
sncosmoParams = cls.sncosmoParamDict(snState, sncosmoModel)
# Now create the class
cls = SNObject(source=snState['ModelSource'])
# Set the SNObject coordinate properties
# Have to be careful to not convert `None` type objects to degrees
setdec, setra = False, False
if snState['_ra'] is not None:
ra = np.degrees(snState['_ra'])
setra = True
if snState['_dec'] is not None:
dec = np.degrees(snState['_dec'])
setdec = True
if setdec and setra:
cls.setCoords(ra, dec)
# Set the SNcosmo parameters
cls.set(**sncosmoParams)
# Set the ebvofMW by hand
cls.ebvofMW = snState['MWE(B-V)']
return cls
@property
def modelOutSideTemporalRange(self):
"""
Defines the behavior of the model when sampled at times beyond the model
definition.
"""
return self._modelOutSideTemporalRange
@modelOutSideTemporalRange.setter
def modelOutSideTemporalRange(self, value):
if value != 'zero':
raise ValueError('Model not implemented, defaulting to zero method\n')
return self._modelOutSideTemporalRange
def equivalentSNCosmoModel(self):
"""
returns an SNCosmo Model which is equivalent to SNObject
"""
snState = self.SNstate
dust = sncosmo.CCM89Dust()
sncosmoModel = sncosmo.Model(source=snState['ModelSource'],
effects=[dust, dust],
effect_names=['host', 'mw'],
effect_frames=['rest', 'obs'])
sncosmoParams = self.sncosmoParamDict(snState, sncosmoModel)
sncosmoParams['mwebv'] = snState['MWE(B-V)']
sncosmoModel.set(**sncosmoParams)
return sncosmoModel
@staticmethod
def equivsncosmoParamDict(SNstate, SNCosmoModel):
"""
return a dictionary that contains the parameters of SNCosmoModel
that are contained in SNstate
Parameters
----------
SNstate : `SNObject.SNstate`, mandatory
Dictionary defining the state of a SNObject
SNCosmoModel : A `sncosmo.Model` instance, mandatory
Returns
-------
sncosmoParams: Dictionary of sncosmo parameters
"""
sncosmoParams = dict()
for param in SNstate:
if param in SNCosmoModel.param_names:
sncosmoParams[param] = SNstate[param]
sncosmoParams['mwebv'] = SNstate['MWE(B-V)']
return sncosmoParams
@staticmethod
def sncosmoParamDict(SNstate, SNCosmoModel):
"""
return a dictionary that contains the parameters of SNCosmoModel
that are contained in SNstate. Note that this does not return the
equivalent SNCosmo model.
Parameters
----------
SNstate : `SNObject.SNstate`, mandatory
Dictionary defining the state of a SNObject
SNCosmoModel : A `sncosmo.Model` instance, mandatory
Returns
-------
sncosmoParams: Dictionary of sncosmo parameters
"""
sncosmoParams = dict()
for param in SNstate:
if param in SNCosmoModel.param_names:
sncosmoParams[param] = SNstate[param]
return sncosmoParams
def summary(self):
'''
summarizes the current state of the SNObject class in a returned
string.
Parameters
----------
None
Returns
-------
Summary State in string format
Examples
--------
>>> t = SNObject()
>>> print t.summary()
'''
state = ' SNObject Summary \n'
state += 'Model = ' + '\n'
state += 'z = ' + str(self.get('z')) + '\n'
state += 'c = ' + str(self.get('c')) + '\n'
state += 'x1 = ' + str(self.get('x1')) + '\n'
state += 'x0 = ' + str(self.get('x0')) + '\n'
state += 't0 = ' + str(self.get('t0')) + '\n'
state += 'ra = ' + str(self._ra) + ' in radians \n'
state += 'dec = ' + str(self._dec) + ' in radians \n'
state += 'MW E(B-V) = ' + str(self.ebvofMW) + '\n'
return state
def setCoords(self, ra, dec):
"""
set the ra and dec coordinate of SNObject to values in radians
corresponding to the given values in degrees
Parameters
----------
ra: float, mandatory
the ra in degrees
dec: float, mandatory
dec in degrees
Returns
-------
None
Examples
--------
>>> t = SNObject()
>>> t.setCoords(ra=30., dec=90.)
>>> t._ra
>>> 0.5235987755982988
>>> t._dec
>>> 1.0471975511965976
"""
if ra is None or dec is None:
raise ValueError('Why try to set coordinates without full'
'coordiantes?\n')
self._ra = np.radians(ra)
self._dec = np.radians(dec)
self.skycoord = np.array([[self._ra], [self._dec]])
self._hascoords = True
return
def set_MWebv(self, value):
"""
if mwebv value is known, this can be used to set the attribute
ebvofMW of the SNObject class to the value (float).
Parameters
----------
value: float, mandatory
value of mw extinction parameter E(B-V) in mags to be used in
applying extinction to the SNObject spectrum
Returns
-------
None
Examples
--------
>>> t = SNObject()
>>> t.set_MWebv(0.)
>>> 0.
"""
self.ebvofMW = value
return
def mwEBVfromMaps(self):
"""
set the attribute ebvofMW of the class from the ra and dec
of the SN. If the ra or dec attribute of the class is None,
set this attribute to None.
Parameters
----------
None
Returns
-------
None
Examples
--------
>>> t = SNObject()
>>> t.setCoords(ra=30., dec=60.)
>>> t.mwEBVfromMaps()
>>> t.ebvofMW
>>> 0.977767825127
.. note:: This function must be run after the class has attributes ra
and dec set. In case it is run before this, the mwebv value
will be set to None.
"""
if not self._hascoords:
raise ValueError('Cannot Calculate EBV from dust maps if ra or dec'
'is `None`')
self.ebvofMW = self.lsstmwebv.calculateEbv(
equatorialCoordinates=self.skycoord)[0]
return
def redshift(self, z, cosmo):
"""
Redshift the instance holding the intrinsic brightness of the object
fixed. By intrinsic brightness here, we mean the BessellB band asbolute
magnitude in rest frame. This requires knowing the cosmology
Parameters
----------
z : float, mandatory
redshift at which the object must be placed.
cosmo : instance of `astropy.cosmology` objects, mandatory
specifies the cosmology.
Returns
-------
None, but it changes the instance
"""
import numbers
# Check that the input redshift is a scalar
try:
assert isinstance(z, numbers.Number)
except:
raise TypeError('The argument z in method redshift should be'
'a scalar Numeric')
# Ensure that the input redshift is greater than 0.
try:
assert z > 0.
except:
raise ValueError('The argument z in the method SNObject.redshift'
'should be greater than 0.')
# Find the current value of the rest frame BessellB AB magnitude
peakAbsMag = self.source_peakabsmag('BessellB', 'AB', cosmo=cosmo)
self.set(z=z)
self.set_source_peakabsmag(peakAbsMag, 'BessellB', 'AB', cosmo=cosmo)
return
def SNObjectSED(self, time, wavelen=None, bandpass=None,
applyExtinction=True):
'''
return a `lsst.sims.photUtils.sed` object from the SN model at the
requested time and wavelengths with or without extinction from MW
according to the SED extinction methods. The wavelengths may be
obtained from a `lsst.sims.Bandpass` object or a `lsst.sims.BandpassDict`
object instead. (Currently, these have the same wavelengths). See notes
for details on handling of exceptions.
If the sed is requested at times outside the validity range of the
model, the flux density is returned as 0. If the time is within the
range of validity of the model, but the wavelength range requested
is outside the range, then the returned fluxes are np.nan outside
the range, and the model fluxes inside
Parameters
----------
time: float
time of observation
wavelen: `np.ndarray` of floats, optional, defaults to None
array containing wavelengths in nm
bandpass: `lsst.sims.photUtils.Bandpass` object or
`lsst.sims.photUtils.BandpassDict`, optional, defaults to `None`.
Using the dict assumes that the wavelength sampling and range
is the same for all elements of the dict.
if provided, overrides wavelen input and the SED is
obtained at the wavelength values native to bandpass
object.
Returns
-------
`sims_photutils.sed` object containing the wavelengths and SED
values from the SN at time time in units of ergs/cm^2/sec/nm
.. note: If both wavelen and bandpassobject are `None` then exception,
will be raised.
Examples
--------
>>> sed = SN.SNObjectSED(time=0., wavelen=wavenm)
'''
if wavelen is None and bandpass is None:
raise ValueError('A non None input to either wavelen or\
bandpassobject must be provided')
# if bandpassobject present, it overrides wavelen
if bandpass is not None:
if isinstance(bandpass, BandpassDict):
firstfilter = bandpass.keys()[0]
bp = bandpass[firstfilter]
else:
bp = bandpass
# remember this is in nm
wavelen = bp.wavelen
flambda = np.zeros(len(wavelen))
# self.mintime() and self.maxtime() are properties describing
# the ranges of SNCosmo.Model in time. Behavior beyond this is
# determined by self.modelOutSideTemporalRange
if (time >= self.mintime()) and (time <= self.maxtime()):
# If SNCosmo is requested a SED value beyond the wavelength range
# of model it will crash. Try to prevent that by returning np.nan for
# such wavelengths. This will still not help band flux calculations
# but helps us get past this stage.
flambda = flambda * np.nan
# Convert to Ang
wave = wavelen * 10.0
mask1 = wave >= self.minwave()
mask2 = wave <= self.maxwave()
mask = mask1 & mask2
wave = wave[mask]
# flux density dE/dlambda returned from SNCosmo in
# ergs/cm^2/sec/Ang, convert to ergs/cm^2/sec/nm
flambda[mask] = self.flux(time=time, wave=wave)
flambda[mask] = flambda[mask] * 10.0
else:
# use prescription for modelOutSideTemporalRange
if self.modelOutSideTemporalRange != 'zero':
raise NotImplementedError('Model not implemented, change to zero\n')
# Else Do nothing as flambda is already 0.
# This takes precedence over being outside wavelength range
if self.rectifySED:
# Note that this converts nans into 0.
flambda = np.where(flambda > 0., flambda, 0.)
SEDfromSNcosmo = Sed(wavelen=wavelen, flambda=flambda)
if not applyExtinction:
return SEDfromSNcosmo
# Apply LSST extinction
global _sn_ax_cache
global _sn_bx_cache
global _sn_ax_bx_wavelen
if _sn_ax_bx_wavelen is None \
or len(wavelen)!=len(_sn_ax_bx_wavelen) \
or (wavelen!=_sn_ax_bx_wavelen).any():
ax, bx = SEDfromSNcosmo.setupCCM_ab()
_sn_ax_cache = ax
_sn_bx_cache = bx
_sn_ax_bx_wavelen = np.copy(wavelen)
else:
ax = _sn_ax_cache
bx = _sn_bx_cache
if self.ebvofMW is None:
raise ValueError('ebvofMW attribute cannot be None Type and must'
' be set by hand using set_MWebv before this'
'stage, or by using setcoords followed by'
'mwEBVfromMaps\n')
SEDfromSNcosmo.addDust(a_x=ax, b_x=bx, ebv=self.ebvofMW)
return SEDfromSNcosmo
def SNObjectSourceSED(self, time, wavelen=None):
"""
Return the rest Frame SED of SNObject at the phase corresponding to
time, at rest frame wavelengths wavelen. If wavelen is None,
then the SED is sampled at the rest frame wavelengths native to the
SALT model being used.
Parameters
----------
time : float, mandatory,
observer frame time at which the SED has been requested in units
of days.
wavelen : `np.ndarray`, optional, defaults to native SALT wavelengths
array of wavelengths in the rest frame of the supernova in units
of nm. If None, this defaults to the wavelengths at which the
SALT model is sampled natively.
Returns
-------
`numpy.ndarray` of dtype float.
.. note: The result should usually match the SALT source spectrum.
However, it may be different for the following reasons:
1. If the time of observation is outside the model range, the values
have to be inserted using additional models. Here only one model
is currently implemented, where outside the model range the value
is set to 0.
2. If the wavelengths are beyond the range of the SALT model, the SED
flambda values are set to `np.nan` and these are actually set to 0.
if `self.rectifySED = True`
3. If the `flambda` values of the SALT model are negative which happens
in the less sampled phases of the model, these values are set to 0,
if `self.rectifySED` = True. (Note: if `self.rectifySED` = True, then
care will be taken to make sure that the flux at 500nm is not exactly
zero, since that will cause PhoSim normalization of the SED to be
NaN).
"""
phase = (time - self.get('t0')) / (1. + self.get('z'))
source = self.source
# Set the default value of wavelength input
if wavelen is None:
# use native SALT grid in Ang
wavelen = source._wave
else:
#input wavelen in nm, convert to Ang
wavelen = wavelen.copy()
wavelen *= 10.0
flambda = np.zeros(len(wavelen))
# self.mintime() and self.maxtime() are properties describing
# the ranges of SNCosmo.Model in time. Behavior beyond this is
# determined by self.modelOutSideTemporalRange
insidephaseRange = (phase <= source.maxphase())and(phase >= source.minphase())
if insidephaseRange:
# If SNCosmo is requested a SED value beyond the wavelength range
# of model it will crash. Try to prevent that by returning np.nan for
# such wavelengths. This will still not help band flux calculations
# but helps us get past this stage.
flambda = flambda * np.nan
mask1 = wavelen >= source.minwave()
mask2 = wavelen <= source.maxwave()
mask = mask1 & mask2
# Where we have to calculate fluxes because it is not `np.nan`
wave = wavelen[mask]
flambda[mask] = source.flux(phase, wave)
else:
if self.modelOutSideTemporalRange == 'zero':
# flambda was initialized as np.zeros before start of
# conditional
pass
else:
raise NotImplementedError('Only modelOutSideTemporalRange=="zero" implemented')
# rectify the flux
if self.rectifySED:
flux = np.where(flambda>0., flambda, 0.)
else:
flux = flambda
# convert per Ang to per nm
flux *= 10.0
# convert ang to nm
wavelen = wavelen / 10.
# If there is zero flux at 500nm, set
# the flux in the slot closest to 500nm
# equal to 0.01*minimum_non_zero_flux
# (this is so SEDs used in PhoSim can have
# finite normalization)
if self.rectifySED:
closest_to_500nm = np.argmin(np.abs(wavelen-500.0))
if flux[closest_to_500nm] == 0.0:
non_zero_flux = np.where(flux>0.0)
if len(non_zero_flux[0])>0:
min_non_zero = np.min(flux[non_zero_flux])
flux[closest_to_500nm] = 0.01*min_non_zero
sed = Sed(wavelen=wavelen, flambda=flux)
# This has the cosmology built in.
return sed
def catsimBandFlux(self, time, bandpassobject):
"""
return the flux in the bandpass in units of maggies which is the flux
the AB magnitude reference spectrum would have in the same band.
Parameters
----------
time: mandatory, float
MJD at which band fluxes are evaluated
bandpassobject: mandatory, `lsst.sims.photUtils.BandPass` object
A particular bandpass which is an instantiation of one of
(u, g, r, i, z, y)
Returns
-------
float value for flux in band in units of maggies
Examples
--------
>>> bandpassnames = ['u', 'g', 'r', 'i', 'z', 'y']
>>> LSST_BandPass = BandpassDict.loadTotalBandpassesFromFiles()
>>> SN = SNObject(ra=30., dec=-60.)
>>> SN.set(z=0.96, t0=571181, x1=2.66, c=0.353, x0=1.796112e-06)
>>> SN.catsimBandFlux(bandpassobject=LSST_BandPass['r'], time=571190.)
>>> 1.9856857972304903e-11
.. note: If there is an unphysical value of sed in
the wavelength range, it produces a flux of `np.nan`
"""
# Speedup for cases outside temporal range of model
if time <= self.mintime() or time >= self.maxtime() :
return 0.
SEDfromSNcosmo = self.SNObjectSED(time=time,
bandpass=bandpassobject)
return SEDfromSNcosmo.calcFlux(bandpass=bandpassobject) / 3631.0
def catsimBandMag(self, bandpassobject, time, fluxinMaggies=None,
noNan=False):
"""
return the magnitude in the bandpass in the AB magnitude system
Parameters
----------
bandpassobject : mandatory,`sims.photUtils.BandPass` instances
LSST Catsim bandpass instance for a particular bandpass
time : mandatory, float
MJD at which this is evaluated
fluxinMaggies: float, defaults to None
provide the flux in maggies, if not provided, this will be evaluated
noNan : Bool, defaults to False
If True, an AB magnitude of 200.0 rather than nan values is
associated with a flux of 0.
Returns
-------
float value of band magnitude in AB system
Examples
--------
"""
if fluxinMaggies is None:
fluxinMaggies = self.catsimBandFlux(bandpassobject=bandpassobject,
time=time)
if noNan:
if fluxinMaggies <= 0.:
return 200.0
with np.errstate(divide='ignore', invalid='ignore'):
return -2.5 * np.log10(fluxinMaggies)
def catsimBandFluxError(self, time, bandpassobject, m5,
fluxinMaggies=None,
magnitude=None,
photParams=None):
"""
return the flux uncertainty in the bandpass in units 'maggies'
(the flux the AB magnitude reference spectrum would have in the
same band.) for a source of given brightness. The source brightness
may be calculated, but the need for calculation is overridden by a
provided flux in bandpass (in units of maggies) which itself may be
overridden by a provided magnitude. If the provided/calculated flux
is 0. or negative the magnitude calculated is taken to be 200.0 rather
than a np.nan.
Parameters
----------
time: mandatory, float
MJD at which band fluxes are evaluated
bandpassobject: mandatory, `lsst.sims.photUtils.BandPass` object
A particular bandpass which is an instantiation of one of
(u, g, r, i, z, y)
m5 : float, mandatory
fiveSigma Depth for the sky observation
photParams : instance of `sims.photUtils.PhotometricParameters`, defaults to `None`
describes the hardware parameters of the Observing system
magnitude : float, defaults to None
AB magnitude of source in bandpass.
fluxinMaggies : float, defaults to None
flux in Maggies for source in bandpass
Returns
-------
float
Examples
--------
.. note: If there is an unphysical value of sed the fluxinMaggies might
be `np.nan`. The magnitude calculated from this is calculated using `noNan`
and is therefore 200.0 rather than `np.nan`.
"""
if fluxinMaggies is None:
fluxinMaggies = self.catsimBandFlux(time=time,
bandpassobject=bandpassobject)
if magnitude is None:
mag = self.catsimBandMag(time=time, fluxinMaggies=fluxinMaggies,
bandpassobject=bandpassobject, noNan=True)
else:
mag = magnitude
# recalculate fluxinMaggies as the previous one might have been `np.nan`
# the noise is contaminated if this is `np.nan`
fluxinMaggies = 10.0**(-0.4 * mag)
if photParams is None:
photParams = PhotometricParameters()
SNR, gamma = calcSNR_m5(magnitude=mag, bandpass=bandpassobject,
m5=m5, photParams=photParams)
return fluxinMaggies / SNR
def catsimBandMagError(self, time, bandpassobject, m5, photParams=None,
magnitude=None):
"""
return the 68 percent uncertainty on the magnitude in the bandpass
Parameters
----------
time: mandatory, float
MJD at which band fluxes are evaluated
bandpassobject: mandatory, `lsst.sims.photUtils.BandPass` object
A particular bandpass which is an instantiation of one of
(u, g, r, i, z, y)
m5 :
photParams :
magnitude :
Returns
-------
float
Examples
--------
.. note: If there is an unphysical value of sed in
the wavelength range, it produces a flux of `np.nan`
"""
if magnitude is None:
mag = self.catsimBandMag(time=time,
bandpassobject=bandpassobject,
noNan=True)
else:
mag = magnitude
bandpass = bandpassobject
if photParams is None:
photParams = PhotometricParameters()
magerr = calcMagError_m5(magnitude=mag,
bandpass=bandpassobject,
m5=m5,
photParams=photParams)
return magerr[0]
def catsimManyBandFluxes(self, time, bandpassDict,
observedBandPassInd=None):
"""
return the flux in the multiple bandpasses of a bandpassDict
indicated by observedBandPassInd in units of maggies
Parameters
----------
time: mandatory, float
MJD at which band fluxes are evaluated
bandpassDict: mandatory, `lsst.sims.photUtils.BandpassDict` instance
observedBandPassInd : optional, list of integers, defaults to None
integer correspdonding to index of the bandpasses used in the
observation in the ordered dict bandpassDict
Returns
-------
`~numpy.ndarray` of length =len(observedBandPassInd)
Examples
--------
.. note: If there is an unphysical value of sed in
the wavelength range, it produces a flux of `np.nan`
"""
SEDfromSNcosmo = self.SNObjectSED(time=time,
bandpass=bandpassDict['u'])
wavelen_step = np.diff(SEDfromSNcosmo.wavelen)[0]
SEDfromSNcosmo.flambdaTofnu()
f = SEDfromSNcosmo.manyFluxCalc(bandpassDict.phiArray,
wavelen_step=wavelen_step,
observedBandpassInd=observedBandPassInd)
return f / 3631.
def catsimManyBandMags(self, time, bandpassDict,
observedBandPassInd=None):
"""
return the flux in the bandpass in units of the flux
the AB magnitude reference spectrum would have in the
same band.
Parameters
----------
time: mandatory, float
MJD at which band fluxes are evaluated
bandpassDict: mandatory, `lsst.sims.photUtils.BandpassDict` instance
observedBandPassInd : optional, list of integers, defaults to None
integer correspdonding to index of the bandpasses used in the
observation in the ordered dict bandpassDict
Returns
-------
`~numpy.ndarray` of length =len(observedBandPassInd)
Examples
--------
.. note: If there is an unphysical value of sed in
the wavelength range, it produces a flux of `np.nan`
"""
f = self.catsimManyBandFluxes(time,
bandpassDict,
observedBandPassInd)
with np.errstate(invalid='ignore', divide='ignore'):
return -2.5 * np.log10(f)
def catsimManyBandADUs(self, time, bandpassDict,
photParams=None,
observedBandPassInds=None):
"""
time: float, mandatory
MJD of the observation
bandpassDict: mandatory,
Dictionary of instances of `sims.photUtils.Bandpass` for
filters
photParams: Instance of `sims.photUtils.PhotometricParameters`, optional,
defaults to None
Describes the observational parameters used in specifying the
photometry of the ovservation
observedBandPassInd: None
Not used now
"""
SEDfromSNcosmo = self.SNObjectSED(time=time,
bandpass=bandpassDict)
bandpassNames = list(bandpassDict.keys())
adus = np.zeros(len(bandpassNames))
for i, filt in enumerate(bandpassNames):
bandpass = bandpassDict[filt]
adus[i] = SEDfromSNcosmo.calcADU(bandpass, photParams=photParams)
return adus
# get the cross-match between the sed fit and cosmoDC2
crossmatch_dex = np.searchsorted(cosmoDC2_data['galaxy_id'], galaxy_id)
np.testing.assert_array_equal(galaxy_id,
cosmoDC2_data['galaxy_id'][crossmatch_dex])
ra = sed_fit_file['ra'][()][subset]
dec = sed_fit_file['dec'][()][subset]
np.testing.assert_array_equal(ra, cosmoDC2_data['ra'][crossmatch_dex])
np.testing.assert_array_equal(dec, cosmoDC2_data['dec'][crossmatch_dex])
# Calculate E(B-V) for dust extinction in Milky Way along relevant
# lines of sight
equatorial_coords = np.array([np.radians(ra), np.radians(dec)])
ebv_model = EBVbase()
ebv_vals = ebv_model.calculateEbv(equatorialCoordinates=equatorial_coords)
ccm_w = None # so that we only have to initialize internal dust once
for i_bp, bp in enumerate('ugrizy'):
fluxes_noMW = {}
fluxes = {}
for component in ['disk', 'bulge']:
fluxes_noMW[component] = np.zeros(n_test_gals, dtype=float)
fluxes[component] = np.zeros(n_test_gals, dtype=float)
for component in ['disk', 'bulge']:
sed_arr = sed_fit_file['%s_sed' % component][()][subset]
av_arr = sed_fit_file['%s_av' % component][()][subset]
rv_arr = sed_fit_file['%s_rv' % component][()][subset]
mn_arr = sed_fit_file['%s_magnorm' %
component][()][i_bp, :][subset]
def calculate_fluxes(in_name, out_name, healpix_id, my_lock):
base_dir = os.path.join('/astro/store/pogo4/danielsf/desc_dc2_truth')
t_start = time.time()
with my_lock as context:
redshift_file_name = os.path.join(base_dir, 'redshift',
'redshift_%d.h5' % healpix_id)
with h5py.File(redshift_file_name, 'r') as in_file:
redshift = in_file['redshift'][()]
galaxy_id_z = in_file['galaxy_id'][()]
with h5py.File(in_name, 'r') as in_file:
sed_names = in_file['sed_names'][()].astype(str)
galaxy_id_disk = in_file['galaxy_id'][()]
sorted_dex = np.argsort(galaxy_id_disk)
galaxy_id_disk = galaxy_id_disk[sorted_dex]
ra = in_file['ra'][()][sorted_dex]
dec = in_file['dec'][()][sorted_dex]
disk_sed = in_file['disk_sed'][()][sorted_dex]
disk_magnorm = in_file['disk_magnorm'][()]
for ii in range(6):
disk_magnorm[ii] = disk_magnorm[ii][sorted_dex]
disk_av = in_file['disk_av'][()][sorted_dex]
disk_rv = in_file['disk_rv'][()][sorted_dex]
#disk_fluxes_in = in_file['disk_fluxes'][()]
#for ii in range(6):
# disk_fluxes_in[ii] = disk_fluxes_in[ii][sorted_dex]
print('ra ', ra.min(), ra.max())
print('dec ', dec.min(), dec.max())
ebv_model = EBVbase()
ebv_arr = ebv_model.calculateEbv(interp=True,
equatorialCoordinates=np.array(
[np.radians(ra),
np.radians(dec)]))
del ebv_model
del ra
del dec
print('loaded disk data in %e' % (time.time() - t_start))
sorted_dex_z = np.argsort(galaxy_id_z)
galaxy_id_z = galaxy_id_z[sorted_dex_z]
redshift = redshift[sorted_dex_z]
if len(galaxy_id_z) != len(galaxy_id_disk):
galaxy_id_z = galaxy_id_z[:len(galaxy_id_disk)]
redshift = redshift[:len(galaxy_id_disk)]
np.testing.assert_array_equal(galaxy_id_z, galaxy_id_disk)
n_threads = 39
d_gal = 50000
mgr = multiprocessing.Manager()
output_dict = mgr.dict()
output_dict['fluxes'] = mgr.list()
output_dict['fluxes_noMW'] = mgr.list()
output_dict['galaxy_id'] = mgr.list()
p_list = []
ct_done = 0
t_start = time.time()
for i_start in range(0, len(disk_av), d_gal):
s = slice(i_start, i_start + d_gal)
p = multiprocessing.Process(target=process_component,
args=(sed_names, ebv_arr[s], redshift[s],
disk_sed[s], disk_av[s], disk_rv[s],
disk_magnorm[:, s], my_lock,
output_dict, galaxy_id_disk[s]))
p.start()
p_list.append(p)
while len(p_list) >= n_threads:
exit_state_list = []
for p in p_list:
exit_state_list.append(p.exitcode)
n_processes = len(p_list)
for ii in range(n_processes - 1, -1, -1):
if exit_state_list[ii] is not None:
p_list.pop(ii)
with my_lock:
ct_done = 0
for chunk in output_dict['galaxy_id']:
ct_done += len(chunk)
duration = (time.time() - t_start) / 3600.0
per = duration / ct_done
prediction = per * len(redshift)
print('ran %e in %.2e hrs; pred %.2e hrs' %
(ct_done, duration, prediction))
for p in p_list:
p.join()
print(output_dict['fluxes'])
disk_fluxes = []
disk_fluxes_noMW = []
for i_bp in range(6):
disk_fluxes.append(
np.concatenate([ff[i_bp] for ff in output_dict['fluxes']]))
disk_fluxes_noMW.append(
np.concatenate([ff[i_bp] for ff in output_dict['fluxes_noMW']]))
disk_fluxes = np.array(disk_fluxes)
disk_fluxes_noMW = np.array(disk_fluxes_noMW)
galaxy_id_disk = np.concatenate(output_dict['galaxy_id'])
sorted_dex = np.argsort(galaxy_id_disk)
galaxy_id_disk = galaxy_id_disk[sorted_dex]
for i_bp in range(6):
assert len(disk_fluxes[i_bp]) == len(galaxy_id_disk)
assert len(disk_fluxes_noMW[i_bp]) == len(galaxy_id_disk)
disk_fluxes[i_bp] = disk_fluxes[i_bp][sorted_dex]
disk_fluxes_noMW[i_bp] = disk_fluxes_noMW[i_bp][sorted_dex]
np.testing.assert_array_equal(galaxy_id_disk, galaxy_id_z)
del output_dict
#for i_bp, bp in enumerate('ugrizy'):
# d_flux_ratio = disk_fluxes_noMW[bp]/disk_fluxes_in[i_bp]
# print(bp,' disk flux ratio ',d_flux_ratio.max(),d_flux_ratio.min())
del disk_sed
del disk_av
del disk_rv
del disk_magnorm
with my_lock as context:
with h5py.File(in_name, 'r') as in_file:
galaxy_id_bulge = in_file['galaxy_id'][()]
sorted_dex = np.argsort(galaxy_id_bulge)
galaxy_id_bulge = galaxy_id_bulge[sorted_dex]
bulge_sed = in_file['bulge_sed'][()][sorted_dex]
bulge_av = in_file['bulge_av'][()][sorted_dex]
bulge_rv = in_file['bulge_rv'][()][sorted_dex]
#bulge_fluxes_in = in_file['bulge_fluxes'][()]
#for ii in range(6):
# bulge_fluxes_in[ii] = bulge_fluxes_in[ii][sorted_dex]
bulge_magnorm = in_file['bulge_magnorm'][()]
for ii in range(6):
bulge_magnorm[ii] = bulge_magnorm[ii][sorted_dex]
np.testing.assert_array_equal(galaxy_id_z, galaxy_id_bulge)
output_dict = mgr.dict()
output_dict['fluxes'] = mgr.list()
output_dict['fluxes_noMW'] = mgr.list()
output_dict['galaxy_id'] = mgr.list()
ct_done = 0
p_list = []
t_start = time.time()
for i_start in range(0, len(bulge_av), d_gal):
s = slice(i_start, i_start + d_gal)
p = multiprocessing.Process(
target=process_component,
args=(sed_names, ebv_arr[s], redshift[s], bulge_sed[s],
bulge_av[s], bulge_rv[s], bulge_magnorm[:, s], my_lock,
output_dict, galaxy_id_bulge[s]))
p.start()
p_list.append(p)
while len(p_list) >= n_threads:
exit_state_list = []
for p in p_list:
exit_state_list.append(p.exitcode)
n_processes = len(p_list)
for ii in range(n_processes - 1, -1, -1):
if exit_state_list[ii] is not None:
p_list.pop(ii)
ct_done += d_gal
duration = (time.time() - t_start) / 3600.0
per = duration / ct_done
prediction = per * len(redshift)
print('ran %e in %.2e hrs; pred %.2e hrs' %
(ct_done, duration, prediction))
for p in p_list:
p.join()
bulge_fluxes = []
bulge_fluxes_noMW = []
for i_bp in range(6):
bulge_fluxes.append(
np.concatenate([ff[i_bp] for ff in output_dict['fluxes']]))
bulge_fluxes_noMW.append(
np.concatenate([ff[i_bp] for ff in output_dict['fluxes_noMW']]))
bulge_fluxes = np.array(bulge_fluxes)
bulge_fluxes_noMW = np.array(bulge_fluxes_noMW)
galaxy_id_bulge = np.concatenate(output_dict['galaxy_id'])
sorted_dex = np.argsort(galaxy_id_bulge)
galaxy_id_bulge = galaxy_id_bulge[sorted_dex]
for i_bp in range(6):
assert len(bulge_fluxes[i_bp]) == len(galaxy_id_bulge)
assert len(bulge_fluxes_noMW[i_bp]) == len(galaxy_id_bulge)
bulge_fluxes[i_bp] = bulge_fluxes[i_bp][sorted_dex]
bulge_fluxes_noMW[i_bp] = bulge_fluxes_noMW[i_bp][sorted_dex]
np.testing.assert_array_equal(galaxy_id_bulge, galaxy_id_z)
del output_dict
#for i_bp, bp in enumerate('ugrizy'):
# bulge_flux_ratio = bulge_fluxes_in[i_bp]/bulge_fluxes_noMW[bp]
# print(bp,' bulge flux ratio ',np.nanmax(bulge_flux_ratio),
# np.nanmin(bulge_flux_ratio))
with my_lock as context:
with h5py.File(out_name, 'w') as out_file:
out_file.create_dataset('galaxy_id', data=galaxy_id_z)
out_file.create_dataset('redshift', data=redshift)
for i_bp, bp in enumerate('ugrizy'):
out_file.create_dataset(
'flux_%s' % bp,
data=1.0e9 * (bulge_fluxes[i_bp] + disk_fluxes[i_bp]))
out_file.create_dataset(
'flux_%s_noMW' % bp,
data=1.0e9 *
(bulge_fluxes_noMW[i_bp] + disk_fluxes_noMW[i_bp]))
#!/usr/bin/env python
import numpy as np
import healpy as hp
from lsst.sims.catUtils.dust import EBVbase
if __name__ == '__main__':
# Read in the Schelgel dust maps and convert them to a healpix map
dustmap = EBVbase()
dustmap.load_ebvMapNorth()
dustmap.load_ebvMapSouth()
# Set up the Healpixel map
nsides = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
for nside in nsides:
lat, ra = hp.pix2ang(nside, np.arange(hp.nside2npix(nside)))
# Move dec to +/- 90 degrees
dec = np.pi/2.0 - lat
ebvMap = dustmap.calculateEbv(equatorialCoordinates=np.array([ra, dec]), interp=False)
np.savez('dust_nside_%s.npz'%nside, ebvMap=ebvMap)
def _process_chunk(db_lock, log_lock, sema, sed_fit_name, cosmoDC2_data,
first_gal, self_dict, bad_gals):
"""
Do all chunk-specific work: compute table contents for a
collection of galaxies and write to db
Parameters
----------
db_lock Used to avoid conflicts writing to sqlite output
log_lock Used to avoid conflicts writing to per-healpixel log
sema A semaphore. Release when done
sed_fit_name File where sed fits for this healpixel are
cosmoDC2_data Values from cosmoDC2 for this healpixel, keyed by
column name
first_gal index of first galaxy in our chunk (in sed fit list)
self_dict Random useful values stored in GalaxyTruthWriter
bad_gals List of galaxy ids, monotone increasing, to be
skipped
"""
dry = self_dict['dry']
chunk_size = self_dict['chunk_size']
dbfile = self_dict['dbfile']
logfile = self_dict['logfile']
if dry:
_logit(
log_lock, logfile,
'_process_chunk invoke for first_gal {}, chunk size {}'.format(
first_gal, chunk_size))
if sema is None:
return
sema.release()
#exit(0)
return
lsst_bp_dict = self_dict['lsst_bp_dict']
galaxy_ids = []
ra = []
dec = []
redshift = []
ebv_vals = None
ebv_vals_init = False # does this belong somewhere else?
ccm_w = None
total_gals = self_dict['total_gals']
chunk_start = first_gal
chunk_end = min(first_gal + chunk_size, total_gals)
with h5py.File(sed_fit_name, 'r') as sed_fit_file:
sed_names = sed_fit_file['sed_names'][()]
sed_names = [s.decode() for s in sed_names] # becse stored as bytes
gals_this_chunk = chunk_end - chunk_start
subset = slice(chunk_start, chunk_end)
galaxy_ids = sed_fit_file['galaxy_id'][()][subset]
to_log = 'Start with galaxy #{}, id={}\n# galaxies for _process_chunk: {}\n'.format(
first_gal, galaxy_ids[0], len(galaxy_ids))
_logit(log_lock, logfile, to_log)
# get the cross-match between the sed fit and cosmoDC2
cosmo_len = len(cosmoDC2_data['galaxy_id'])
crossmatch_dex = np.searchsorted(cosmoDC2_data['galaxy_id'],
galaxy_ids)
np.testing.assert_array_equal(
galaxy_ids, cosmoDC2_data['galaxy_id'][crossmatch_dex])
ra = sed_fit_file['ra'][()][subset]
dec = sed_fit_file['dec'][()][subset]
np.testing.assert_array_equal(ra, cosmoDC2_data['ra'][crossmatch_dex])
np.testing.assert_array_equal(dec,
cosmoDC2_data['dec'][crossmatch_dex])
good_ixes = _good_indices(galaxy_ids.tolist(), bad_gals[0])
if (len(good_ixes) == 0):
if sema is not None:
sema.release()
return
else:
_logit(
log_lock, logfile,
'Found {} good indices for chunk starting with {}\n'.format(
len(good_ixes), chunk_start))
flux_by_band_MW = {}
flux_by_band_noMW = {}
# Calculate E(B-V) for dust extinction in Milky Way along relevant
# lines of sight
band_print = "Processing band {}, first gal {}, time {}\n"
if not ebv_vals_init:
equatorial_coords = np.array([np.radians(ra), np.radians(dec)])
ebv_model = EBVbase()
ebv_vals = ebv_model.calculateEbv(
equatorialCoordinates=equatorial_coords, interp=True)
ebv_vals_init = True
for i_bp, bp in enumerate('ugrizy'):
if (i_bp == 0 or i_bp == 5):
_logit(log_lock, logfile,
band_print.format(bp, first_gal, dt.now()))
fluxes_noMW = {}
fluxes = {}
for component in ['disk', 'bulge']:
fluxes_noMW[component] = np.zeros(gals_this_chunk, dtype=float)
fluxes[component] = np.zeros(gals_this_chunk, dtype=float)
for component in ['disk', 'bulge']:
#print(" Processing component ", component)
sed_arr = sed_fit_file['%s_sed' % component][()][subset]
av_arr = sed_fit_file['%s_av' % component][()][subset]
rv_arr = sed_fit_file['%s_rv' % component][()][subset]
mn_arr = sed_fit_file['%s_magnorm' %
component][()][i_bp, :][subset]
z_arr = cosmoDC2_data['redshift'][crossmatch_dex]
gii = 0
done = False
for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate(
zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)):
if done: break
while good_ixes[gii] < i_gal:
gii += 1
if gii == len(good_ixes): # ran out of good ones
done = True
break
if done: break
if good_ixes[gii] > i_gal: # skipped over it; it's bad
continue
# Leave space for it in the arrays, but values
# for all the fluxes will be left at 0
# read in the SED file from the library
sed_file_name = os.path.join(self_dict['sed_lib_dir'],
sed_names[s_dex])
sed = sims_photUtils.Sed()
sed.readSED_flambda(sed_file_name)
# find and apply normalizing flux
fnorm = sims_photUtils.getImsimFluxNorm(sed, mn)
sed.multiplyFluxNorm(fnorm)
# add internal dust
if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w):
ccm_w = np.copy(sed.wavelen)
a_x, b_x = sed.setupCCM_ab()
sed.addDust(a_x, b_x, A_v=av, R_v=rv)
# apply redshift
sed.redshiftSED(zz, dimming=True)
# flux, in Janskys, without Milky Way dust extinction
f_noMW = sed.calcFlux(lsst_bp_dict[bp])
# apply Milky Way dust
# (cannot reuse a_x, b_x because wavelength grid changed
# when we called redshiftSED)
a_x_mw, b_x_mw = sed.setupCCM_ab()
sed.addDust(a_x_mw, b_x_mw, R_v=3.1, ebv=ebv)
f_MW = sed.calcFlux(lsst_bp_dict[bp])
fluxes_noMW[component][i_gal] = f_noMW
fluxes[component][i_gal] = f_MW
if (component == 'disk') and (bp == 'r'):
redshift = z_arr
# Sum components and convert to nanojansky
total_fluxes = (fluxes_noMW['disk'] + fluxes_noMW['bulge']) * 10**9
total_fluxes_MW = (fluxes['disk'] + fluxes['bulge']) * 10**9
dummy_sed = sims_photUtils.Sed()
# add magnification due to weak lensing
kappa = cosmoDC2_data['convergence'][crossmatch_dex]
gamma_sq = (cosmoDC2_data['shear_1'][crossmatch_dex]**2 +
cosmoDC2_data['shear_2'][crossmatch_dex]**2)
magnification = 1.0 / ((1.0 - kappa)**2 - gamma_sq)
magnified_fluxes = magnification * total_fluxes
magnified_fluxes_MW = magnification * total_fluxes_MW
flux_by_band_noMW[bp] = magnified_fluxes
flux_by_band_MW[bp] = magnified_fluxes_MW
# Open connection to sqlite db and write
#print('Time before db write is {}, first gal={}'.format(dt.now(), first_gal))
#sys.stdout.flush()
if not db_lock.acquire(timeout=120.0):
_logit(log_lock, logfile, "Failed to acquire db lock, first gal=",
first_gal)
if sema is None:
return
sema.release()
exit(1)
try:
_write_sqlite(dbfile, galaxy_ids, ra, dec, redshift, flux_by_band_MW,
flux_by_band_noMW, good_ixes)
db_lock.release()
if sema is not None:
sema.release()
_logit(
log_lock, logfile,
'Time after db write: {}, first_gal={}\n'.format(
dt.now(), first_gal))
exit(0)
except Exception as ex:
db_lock.release()
if sema is not None:
sema.release()
raise (ex)
#!/usr/bin/env python
import numpy as np
import healpy as hp
from lsst.sims.catUtils.dust import EBVbase
if __name__ == '__main__':
# Read in the Schelgel dust maps and convert them to a healpix map
dustmap = EBVbase()
dustmap.load_ebvMapNorth()
dustmap.load_ebvMapSouth()
# Set up the Healpixel map
nsides = [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024]
for nside in nsides:
lat, ra = hp.pix2ang(nside, np.arange(hp.nside2npix(nside)))
# Move dec to +/- 90 degrees
dec = np.pi / 2.0 - lat
ebvMap = dustmap.calculateEbv(equatorialCoordinates=np.array([ra,
dec]),
interp=False)
np.savez('dust_nside_%s.npz' % nside, ebvMap=ebvMap)
star_r = star_cmd.fetchmany(chunk_size)
tot_stars = 0
while len(star_r) > 0:
n_stars = len(star_r)
tot_stars += n_stars
print('n_stars %d tot %d of %d' %
(n_stars, tot_stars, final_ct))
star_r = np.array(star_r).transpose()
unq = star_r[0].astype(int) * 1024 + 2
ra = star_r[1].astype(float)
dec = star_r[2].astype(float)
sed_name = star_r[3].astype(str)
mag_norm = star_r[4].astype(float)
eq_coord = np.array([np.radians(ra), np.radians(dec)])
ebv = ebv_gen.calculateEbv(equatorialCoordinates=eq_coord,
interp=True)
delta_ast = rng.normal(0.0, ast_err_rad, n_stars)
(ra_smeared,
dec_smeared) = smear_ra_dec(ra, dec, delta_ast, rng)
delta_mag = rng.normal(0.0, phot_err, size=(n_stars, 6))
mags = np.zeros((n_stars, 6), dtype=float)
for ii in range(n_stars):
spec = photUtils.Sed()
file_name = os.path.join(sed_dir,
defaultSpecMap[sed_name[ii]])
spec.readSED_flambda(file_name)
if not np.array_equal(spec.wavelen, dust_wav):
a_x, b_x = spec.setupCCMab()
def process_agn_chunk(chunk, filter_obs, mjd_obs, m5_obs,
coadd_m5, m5_single,
obs_md_list, proper_chip, out_data,
lock):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
agn_model = ExtraGalacticVariabilityModels()
dust_model = EBVbase()
with h5py.File('data/ebv_grid.h5', 'r') as in_file:
ebv_grid = in_file['ebv_grid'].value
extinction_grid = in_file['extinction_grid'].value
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None]*n_t
params = {}
params['agn_sfu'] = chunk['agn_sfu']
params['agn_sfg'] = chunk['agn_sfg']
params['agn_sfr'] = chunk['agn_sfr']
params['agn_sfi'] = chunk['agn_sfi']
params['agn_sfz'] = chunk['agn_sfz']
params['agn_sfy'] = chunk['agn_sfy']
params['agn_tau'] = chunk['agn_tau']
params['seed'] = chunk['id']+1
ebv = dust_model.calculateEbv(equatorialCoordinates=np.array([np.radians(chunk['ra']),
np.radians(chunk['dec'])]),
interp=True)
for i_bp, bp in enumerate('ugrizy'):
extinction_values = np.interp(ebv, ebv_grid, extinction_grid[i_bp])
chunk['%s_ab' % bp] += extinction_values
chunk['AGNLSST%s' % bp] += extinction_values
dmag = agn_model.applyAgn(np.where(np.array([True]*len(chunk))),
params, mjd_obs,
redshift=chunk['redshift'])
dmag_mean = np.mean(dmag, axis=2)
assert dmag_mean.shape == (6,n_obj)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_gal = np.zeros((6,n_obj), dtype=float)
flux_agn_q = np.zeros((6,n_obj), dtype=float)
flux_coadd = np.zeros((6,n_obj), dtype=float)
mag_coadd = np.zeros((6,n_obj), dtype=float)
snr_coadd = np.zeros((6,n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1,
exptime=30.0)
t_start_snr = time.time()
for i_bp, bp in enumerate('ugrizy'):
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0*coadd_visits[bp])
flux_gal[i_bp] = dummy_sed.fluxFromMag(chunk['%s_ab' % bp])
flux_agn_q[i_bp] = dummy_sed.fluxFromMag(chunk['AGNLSST%s' % bp] +
dmag_mean[i_bp,:])
flux_coadd[i_bp] = flux_gal[i_bp]+flux_agn_q[i_bp]
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp],
gamma) = SNR.calcSNR_m5(mag_coadd[i_bp],
lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp],
gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp],
m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
n_bp = len(valid_obs[0])
if n_bp == 0:
continue
mag0_arr = chunk['AGNLSST%s' % bp]
dmag_bp = dmag[i_bp][:,valid_obs[0]]
assert dmag_bp.shape == (n_obj, n_bp)
agn_flux_tot = dummy_sed.fluxFromMag(mag0_arr[:,None]+dmag_bp)
q_flux = flux_agn_q[i_bp]
agn_dflux = np.abs(agn_flux_tot-q_flux[:,None])
flux_tot = flux_gal[i_bp][:, None] + agn_flux_tot
assert flux_tot.shape == (n_obj, n_bp)
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot,
snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp]
noise_single = flux_tot/snr_single_val
noise = np.sqrt(noise_coadd[:,None]**2 + noise_single**2)
dflux_thresh = 5.0*noise
detected = (agn_dflux>=dflux_thresh)
assert detected.shape == (n_obj, n_bp)
snr_arr[:,valid_obs[0]] = agn_dflux/noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj, valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['dec'],
photometry_mask_1d, obs_md_list,
filter_obs, proper_chip)
duration = (time.time()-t_before_chip)/3600.0
unq_out = -1*np.ones(n_obj, dtype=int)
mjd_out = -1.0*np.ones(n_obj, dtype=float)
snr_out = -1.0*np.ones(n_obj, dtype=float)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:]
if detected.any():
unq_out[i_obj] = chunk['galtileid'][i_obj]
first_dex = np.where(detected)[0].min()
mjd_out[i_obj] = mjd_obs[first_dex]
snr_out[i_obj] = snr_arr[i_obj, first_dex]
valid = np.where(unq_out>=0)
unq_out = unq_out[valid]
mjd_out = mjd_out[valid]
snr_out = snr_out[valid]
with lock:
existing_keys = list(out_data.keys())
if len(existing_keys) == 0:
key_val = 0
else:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None)
out_data[key_val] = (unq_out, mjd_out, snr_out)
def process_sne_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, m5_single,
obs_md_list, proper_chip, invisible_set, out_data, lock):
sne_interp_file = 'data/sne_interp_models.h5'
global _ct_sne
n_t = len(filter_obs)
n_obj = len(chunk)
with h5py.File('data/ebv_grid.h5', 'r') as in_file:
ebv_grid = in_file['ebv_grid'].value
extinction_grid = in_file['extinction_grid'].value
dust_model = EBVbase()
ebv = dust_model.calculateEbv(equatorialCoordinates=np.array(
[np.radians(chunk['ra']),
np.radians(chunk['dec'])]),
interp=True)
for i_bp, bp in enumerate('ugrizy'):
vv = np.interp(ebv, ebv_grid, extinction_grid[i_bp])
chunk['A_%s' % bp] = vv
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None] * n_t
n_t_per_filter = {}
t_obs_arr = {}
i_obs_per_filter = {}
for i_bp, bp in enumerate('ugrizy'):
valid = np.where(filter_obs == i_bp)
n_t_per_filter[bp] = len(valid[0])
i_obs_per_filter[bp] = valid[0]
if n_t_per_filter[bp] == 0:
continue
t_obs_arr[bp] = mjd_obs[valid]
# first just need to interpolate some stuff
ct_invis = 0
d_mag = np.zeros((n_obj, n_t), dtype=float)
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
with h5py.File(sne_interp_file, 'r') as in_file:
param_mins = in_file['param_mins'].value
d_params = in_file['d_params'].value
t_grid = in_file['t_grid'].value
abs_mag_0 = param_mins[3]
i_x_arr = np.round(
(chunk['x1'] - param_mins[0]) / d_params[0]).astype(int)
i_c_arr = np.round(
(chunk['c0'] - param_mins[1]) / d_params[1]).astype(int)
i_z_arr = np.round(
(chunk['redshift'] - param_mins[2]) / d_params[2]).astype(int)
model_tag = i_x_arr + 100 * i_c_arr + 10000 * i_z_arr
unq_tag = np.unique(model_tag)
for i_tag in unq_tag:
valid_obj = np.where(model_tag == i_tag)
if i_tag in invisible_set:
ct_invis += len(valid_obj[0])
continue
d_abs_mag = chunk['abs_mag'][valid_obj] - abs_mag_0
mag_grid = in_file['%d' % i_tag].value
for i_bp, bp in enumerate('ugrizy'):
if n_t_per_filter[bp] == 0:
continue
valid_obs = i_obs_per_filter[bp]
assert len(valid_obs) == len(t_obs_arr[bp])
t_matrix = t_obs_arr[bp] - chunk['t0'][valid_obj, None]
t_arr = t_matrix.flatten()
sne_mag = np.interp(t_arr, t_grid, mag_grid[i_bp]).reshape(
(len(valid_obj[0]), n_t_per_filter[bp]))
for ii in range(len(valid_obj[0])):
obj_dex = valid_obj[0][ii]
sne_mag[ii] += d_abs_mag[ii] + chunk['A_%s' % bp][ii]
d_mag[obj_dex, valid_obs] = sne_mag[ii] - m5_single[bp]
photometry_mask[obj_dex,
valid_obs] = sne_mag[ii] < m5_single[bp]
for i_obj in range(n_obj):
if photometry_mask[i_obj, :].any():
photometry_mask_1d[i_obj] = True
n_unq = len(unq_tag)
ct_detected = 0
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['dec'],
photometry_mask_1d, obs_md_list, filter_obs,
proper_chip)
duration = (time.time() - t_before_chip) / 3600.0
unq_arr = -1 * np.ones(n_obj, dtype=int)
mjd_arr = -1.0 * np.ones(n_obj, dtype=float)
redshift_arr = -1.0 * np.ones(n_obj, dtype=float)
snr_arr = -1.0 * np.ones(n_obj, dtype=float)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj, :] & chip_mask[i_obj, :]
if detected.any():
unq_arr[i_obj] = chunk['galtileid'][i_obj]
first_dex = np.where(detected)[0].min()
snr_arr[i_obj] = 5.0 * 10**(-0.4 * (d_mag[i_obj, first_dex]))
mjd_arr[i_obj] = mjd_obs[first_dex]
redshift_arr[i_obj] = chunk['redshift'][i_obj]
valid = np.where(unq_arr >= 0)
unq_arr = unq_arr[valid]
mjd_arr = mjd_arr[valid]
snr_arr = snr_arr[valid]
redshift_arr = redshift_arr[valid]
with lock:
existing_keys = list(out_data.keys())
if len(existing_keys) == 0:
key_val = 0
else:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None, None)
out_data[key_val] = (unq_arr, mjd_arr, snr_arr, redshift_arr)