def calculate_mags(sed_name, mag_norm, out_dict):
"""
Parameters
----------
sed_name is a numpy array of SED names
mag_norm is a numpy array of magNorms
out_dict is a multiprocessing.Manager.dict() that will
store the magnitudes calculated by this process.
"""
i_process = mp.current_process().pid
bp_dir = getPackageDir('throughputs')
bp_dir = os.path.join(bp_dir, 'imsim', 'goal')
bp_dict = BandpassDict.loadTotalBandpassesFromFiles(bandpassDir=bp_dir)
sed_dir = getPackageDir('sims_sed_library')
out_mags = np.zeros((len(sed_name), 6), dtype=float)
for i_star, (s_name, m_norm) in enumerate(zip(sed_name, mag_norm)):
spec = Sed()
spec.readSED_flambda(os.path.join(sed_dir, defaultSpecMap[s_name]))
fnorm = getImsimFluxNorm(spec, m_norm)
spec.multiplyFluxNorm(fnorm)
mags = bp_dict.magListForSed(spec)
out_mags[i_star] = mags
out_dict[i_process] = out_mags
def test_norm(self):
"""
Test that the special test case getImsimFluxNorm
returns the same value as calling calcFluxNorm actually
passing in the imsim Bandpass
"""
bp = Bandpass()
bp.imsimBandpass()
rng = np.random.RandomState(1123)
wavelen = np.arange(300.0, 2000.0, 0.17)
for ix in range(10):
flux = rng.random_sample(len(wavelen))*100.0
sed = Sed()
sed.setSED(wavelen=wavelen, flambda=flux)
magmatch = rng.random_sample()*5.0 + 10.0
control = sed.calcFluxNorm(magmatch, bp)
test = getImsimFluxNorm(sed, magmatch)
# something about how interpolation is done in Sed means
# that the values don't come out exactly equal. They come
# out equal to 8 seignificant digits, though.
self.assertEqual(control, test)
def _fluxes(sed_name, mag_norm, redshift):
"""
Find the fluxes for a galaxy component
Parameters
----------
sed_name is an SED file name
mag_norm is a float
redshift is a float
Returns
-------
array of fluxes in ugrizy order
"""
if not hasattr(_fluxes, '_bp_dict'):
bp_dir = getPackageDir('throughputs')
bp_dir = os.path.join(bp_dir, 'imsim', 'goal')
_fluxes._bp_dict = BandpassDict.loadTotalBandpassesFromFiles(
bandpassDir=bp_dir)
_fluxes._sed_dir = getPackageDir('sims_sed_library')
spec = Sed()
full_sed_name = os.path.join(_fluxes._sed_dir, sed_name)
if not os.path.isfile(full_sed_name):
full_sed_name = os.path.join(_fluxes._sed_dir,
defaultSpecMap[sed_name])
spec.readSED_flambda(full_sed_name)
fnorm = getImsimFluxNorm(spec, mag_norm)
spec.multiplyFluxNorm(fnorm)
spec.redshiftSED(redshift, dimming=True)
return _fluxes._bp_dict.fluxListForSed(spec)
def test_norm(self):
"""
Test that the special test case getImsimFluxNorm
returns the same value as calling calcFluxNorm actually
passing in the imsim Bandpass
"""
bp = Bandpass()
bp.imsimBandpass()
rng = np.random.RandomState(1123)
wavelen = np.arange(300.0, 2000.0, 0.17)
for ix in range(10):
flux = rng.random_sample(len(wavelen)) * 100.0
sed = Sed()
sed.setSED(wavelen=wavelen, flambda=flux)
magmatch = rng.random_sample() * 5.0 + 10.0
control = sed.calcFluxNorm(magmatch, bp)
test = getImsimFluxNorm(sed, magmatch)
# something about how interpolation is done in Sed means
# that the values don't come out exactly equal. They come
# out equal to 8 seignificant digits, though.
self.assertEqual(control, test)
def _compute_SED(self):
self._sed_obj = sims_photUtils.Sed()
self._sed_obj.readSED_flambda(self.sed_file)
fnorm = sims_photUtils.getImsimFluxNorm(self._sed_obj, self.mag_norm)
self._sed_obj.multiplyFluxNorm(fnorm)
if self.iAv != 0:
self.shared_resources['ccm_model'].add_dust(
self._sed_obj, self.iAv, self.iRv, 'intrinsic')
if self.redshift != 0:
self._sed_obj.redshiftSED(self.redshift, dimming=True)
self._sed_obj.resampleSED(wavelen_match=self.bp_dict.wavelenMatch)
if self.gAv != 0:
self.shared_resources['ccm_model'].add_dust(
self._sed_obj, self.gAv, self.gRv, 'Galactic')
def __init__(self, chunk):
"""
chunk = (simobjid, hpid, sedFilename, magNorm, ebv, varParamStr,
parallax, ra, dec)
"""
# initialize some member variables expected by the
# stellar variability models
self.photParams = sims_photUtils.PhotometricParameters(nexp=1,
exptime=30)
self.lsstBandpassDict = sims_photUtils.BandpassDict.loadTotalBandpassesFromFiles(
)
self._actually_calculated_columns = []
for bp in 'ugrizy':
self._actually_calculated_columns.append('lsst_%s' % bp)
sed_dir = os.environ['SIMS_SED_LIBRARY_DIR']
self.quiescent_mags = {}
for bp in 'ugrizy':
self.quiescent_mags[bp] = np.zeros(len(chunk), dtype=float)
self.parallax = np.zeros(len(chunk), dtype=float)
self.ebv = np.zeros(len(chunk), dtype=float)
self.varParamStr = np.empty(len(chunk), dtype=(str, 300))
self.simobjid = np.empty(len(chunk), dtype=int)
ccm_wav = None
# find the quiescent magnitudes of the stars
for i_star, star in enumerate(chunk):
self.simobjid[i_star] = int(star[0])
full_file_name = os.path.join(sed_dir, defaultSpecMap[star[2]])
spec = sims_photUtils.Sed()
spec.readSED_flambda(full_file_name)
fnorm = sims_photUtils.getImsimFluxNorm(spec, float(star[3]))
spec.multiplyFluxNorm(fnorm)
if ccm_wav is None or not np.array_equal(spec.wavelen, ccm_wav):
ccm_wav = np.copy(spec.wavelen)
a_x, b_x = spec.setupCCM_ab()
ebv_val = float(star[4])
spec.addDust(a_x, b_x, ebv=ebv_val, R_v=3.1)
mag_list = self.lsstBandpassDict.magListForSed(spec)
for i_bp, bp in enumerate('ugrizy'):
self.quiescent_mags[bp][i_star] = mag_list[i_bp]
self.ebv[i_star] = ebv_val
self.varParamStr[i_star] = star[5]
self.parallax[i_star] = float(star[6])
self.parallax *= (np.pi / 648000000.0) # convert from mas to radians
def _create_sed(self):
"""
Function to create the lsst.sims.photUtils Sed object.
"""
self.sed = sims_photUtils.Sed()
self.sed.readSED_flambda(self.sed_file)
fnorm = sims_photUtils.getImsimFluxNorm(self.sed, self.mag_norm)
self.sed.multiplyFluxNorm(fnorm)
if self.iAv != 0:
self.add_dust(self.iAv, self.iRv, 'intrinsic')
if self.redshift != 0:
self.sed.redshiftSED(self.redshift, dimming=True)
self.sed.resampleSED(wavelen_match=self.bp_dict.wavelenMatch)
if self.gAv != 0:
self.add_dust(self.gAv, self.gRv, 'Galactic')
def process_component(sed_names, ebv_arr, redshift, sed_dexes, av, rv, magnorm,
my_lock, output_dict, galaxy_id):
(dummy, hw_bp_dict) = sims_photUtils.BandpassDict.loadBandpassesFromFiles()
n_obj = len(sed_dexes)
fluxes = np.zeros((6, n_obj), dtype=float)
fluxes_noMW = np.zeros((6, n_obj), dtype=float)
t_start = time.time()
for sed_id in np.unique(sed_dexes):
valid = np.where(sed_dexes == sed_id)
sed_full_name = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'],
sed_names[sed_id])
spec_rest = sims_photUtils.Sed()
spec_rest.readSED_flambda(sed_full_name, cache_sed=False)
a_x_rest, b_x_rest = spec_rest.setupCCM_ab()
fnorm_at_21 = sims_photUtils.getImsimFluxNorm(spec_rest, 21.0)
for g_dex in valid[0]:
spec_rest_dust = sims_photUtils.Sed(wavelen=spec_rest.wavelen,
flambda=spec_rest.flambda)
spec_rest_dust.addDust(a_x_rest,
b_x_rest,
A_v=av[g_dex],
R_v=rv[g_dex])
spec_rest_dust.redshiftSED(redshift[g_dex], dimming=True)
flux_list_noMW = hw_bp_dict.fluxListForSed(spec_rest_dust)
a_x, b_x = spec_rest_dust.setupCCM_ab()
spec_rest_dust.addDust(a_x, b_x, R_v=3.1, ebv=ebv_arr[g_dex])
flux_list = hw_bp_dict.fluxListForSed(spec_rest_dust)
for i_bp, bp in enumerate('ugrizy'):
factor = fnorm_at_21 * np.power(
10.0, -0.4 * (magnorm[i_bp][g_dex] - 21.0))
fluxes[i_bp][g_dex] = factor * flux_list[i_bp]
fluxes_noMW[i_bp][g_dex] = factor * flux_list_noMW[i_bp]
with my_lock as context:
print('adding %d to fluxes' % len(fluxes))
output_dict['fluxes'].append(fluxes)
output_dict['fluxes_noMW'].append(fluxes_noMW)
output_dict['galaxy_id'].append(galaxy_id)
def add_flux(self, sed_name, redshift, magnorm_dict, av, rv, bp_dict=None):
if bp_dict is None:
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
result_dict_no_mw = {}
result_dict_mw = {}
sed_obj = Sed()
sed_obj.readSED_flambda(os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], sed_name))
a_x, b_x = sed_obj.setupCCM_ab()
for bandpass_name in bp_dict.keys():
sed_copy = deepcopy(sed_obj)
flux_norm = getImsimFluxNorm(sed_copy, magnorm_dict[bandpass_name])
sed_copy.multiplyFluxNorm(flux_norm)
sed_copy.redshiftSED(redshift, dimming=True)
band_flux = sed_copy.calcFlux(bp_dict[bandpass_name])
result_dict_no_mw[bandpass_name] = band_flux
sed_copy.addDust(a_x, b_x, A_v=av, R_v=rv)
band_flux_mw = sed_copy.calcFlux(bp_dict[bandpass_name])
result_dict_mw[bandpass_name] = band_flux_mw
return result_dict_no_mw, result_dict_mw
query += 'galaxy WHERE varParamStr IS NOT NULL '
query += 'AND dec BETWEEN -2.5 AND 2.5 '
query += 'AND (ra<2.5 OR ra>357.5)'
dtype = np.dtype([('magnorm', float), ('redshift', float),
('varParamStr', str, 400)])
data_iter = db.get_arbitrary_chunk_iterator(query,
dtype=dtype,
chunk_size=10000)
with open('data/dc1_agn_params.txt', 'w') as out_file:
out_file.write('# z m_i M_i tau sfu sfg sfr sfi sfz sfy\n')
for chunk in data_iter:
DM = cosmo.distanceModulus(redshift=chunk['redshift'])
k_corr = np.interp(chunk['redshift'], z_grid, k_grid)
for i_row, agn in enumerate(chunk):
ss = Sed(wavelen=base_sed.wavelen, flambda=base_sed.flambda)
fnorm = getImsimFluxNorm(ss, agn['magnorm'])
ss.multiplyFluxNorm(fnorm)
ss.redshiftSED(agn['redshift'], dimming=True)
mag = ss.calcMag(bp)
abs_m_i = mag - DM[i_row] - k_corr[i_row]
params = json.loads(agn['varParamStr'])['pars']
out_file.write(
'%e %e %e %e %e %e %e %e %e %e\n' %
(agn['redshift'], mag, abs_m_i, params['agn_tau'],
params['agn_sfu'], params['agn_sfg'], params['agn_sfr'],
params['agn_sfi'], params['agn_sfz'], params['agn_sfy']))
def loadSedsFromList(self, sedNameList, magNormList, \
internalAvList=None, galacticAvList=None, redshiftList=None):
"""
Load the Seds specified by sedNameList, applying the specified normalization,
extinction, and redshift.
@param [in] sedList is a list of file names containing Seds
@param [in] magNorm is the magnitude normalization
@param [in] internalAvList is an optional list of A(V) due to internal
dust
@param [in] galacticAvList is an optional list of A(V) due to
Milky Way dust
@param [in] redshiftList is an optional list of redshifts for the
input Sed
Seds are read in and stored to this object's internal list of Seds.
Note: if you constructed this SedList object without internalAvList,
you cannot load Seds with internalAvList now. Likewise for galacticAvlist
and redshiftList.
"""
if not self._initialized:
if internalAvList is not None:
self._internal_av_list = copy.deepcopy(list(internalAvList))
else:
self._internal_av_list = None
if galacticAvList is not None:
self._galactic_av_list = copy.deepcopy(list(galacticAvList))
else:
self._galactic_av_list = None
if redshiftList is not None:
self._redshift_list = copy.deepcopy(list(redshiftList))
else:
self._redshift_list = None
else:
if self._internal_av_list is None and internalAvList is not None:
raise RuntimeError(
"This SedList does not contain internalAvList")
elif self._internal_av_list is not None:
if internalAvList is None:
self._internal_av_list += [None] * len(sedNameList)
else:
self._internal_av_list += list(internalAvList)
if self._galactic_av_list is None and galacticAvList is not None:
raise RuntimeError(
"This SedList does not contain galacticAvList")
elif self._galactic_av_list is not None:
if galacticAvList is None:
self._galactic_av_list += [None] * len(sedNameList)
else:
self._galactic_av_list += list(galacticAvList)
if self._redshift_list is None and redshiftList is not None:
raise RuntimeError(
"This SedList does not contain redshiftList")
elif self._redshift_list is not None:
if redshiftList is None:
self._redshift_list += [None] * len(sedNameList)
else:
self._redshift_list += list(redshiftList)
temp_sed_list = []
for sedName, magNorm in zip(sedNameList, magNormList):
sed = Sed()
if sedName != "None":
if self._spec_map is not None:
sed.readSED_flambda(
os.path.join(self._file_dir, self._spec_map[sedName]))
else:
sed.readSED_flambda(os.path.join(self._file_dir, sedName))
if self._normalizing_bandpass is not None:
fNorm = sed.calcFluxNorm(magNorm,
self._normalizing_bandpass)
else:
fNorm = getImsimFluxNorm(sed, magNorm)
sed.multiplyFluxNorm(fNorm)
temp_sed_list.append(sed)
if internalAvList is not None:
self._av_int_wavelen, \
self._a_int, \
self._b_int = self.applyAv(temp_sed_list, internalAvList,
self._av_int_wavelen, self._a_int, self._b_int)
if redshiftList is not None:
self.applyRedshift(temp_sed_list, redshiftList)
if self._wavelen_match is not None:
for sedObj in temp_sed_list:
if sedObj.wavelen is not None:
sedObj.resampleSED(wavelen_match=self._wavelen_match)
if galacticAvList is not None:
self._av_gal_wavelen, \
self._a_gal, \
self._b_gal = self.applyAv(temp_sed_list, galacticAvList,
self._av_gal_wavelen, self._a_gal, self._b_gal)
self._sed_list += temp_sed_list
self._initialized = True
size=n_to_validate)
else:
dexes_to_validate = np.arange(len(control_qties['redshift']),
dtype=int)
max_offset = -1.0
sed_dir = os.environ['SIMS_SED_LIBRARY_DIR']
tot_bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
for ii in dexes_to_validate:
disk_name = os.path.join(sed_dir,
sed_names[disk_sed_idx[ii]].decode())
bulge_name = os.path.join(sed_dir,
sed_names[bulge_sed_idx[ii]].decode())
for i_bp, bp in enumerate('ugrizy'):
disk_s = Sed()
disk_s.readSED_flambda(disk_name)
fnorm = getImsimFluxNorm(disk_s, disk_magnorm[i_bp][ii])
disk_s.multiplyFluxNorm(fnorm)
ax, bx = disk_s.setupCCM_ab()
disk_s.addDust(ax, bx, A_v=disk_av[ii], R_v=disk_rv[ii])
disk_s.redshiftSED(control_qties['redshift'][ii], dimming=True)
disk_f = disk_s.calcFlux(tot_bp_dict[bp])
bulge_s = Sed()
bulge_s.readSED_flambda(bulge_name)
fnorm = getImsimFluxNorm(bulge_s, bulge_magnorm[i_bp][ii])
bulge_s.multiplyFluxNorm(fnorm)
ax, bx = bulge_s.setupCCM_ab()
bulge_s.addDust(ax, bx, A_v=bulge_av[ii], R_v=bulge_rv[ii])
bulge_s.redshiftSED(control_qties['redshift'][ii],
dimming=True)
bulge_f = bulge_s.calcFlux(tot_bp_dict[bp])
sed_arr = sed_fit_file['%s_sed' % component][()][subset]
av_arr = sed_fit_file['%s_av' % component][()][subset]
rv_arr = sed_fit_file['%s_rv' % component][()][subset]
mn_arr = sed_fit_file['%s_magnorm' %
component][()][i_bp, :][subset]
z_arr = cosmoDC2_data['redshift'][crossmatch_dex]
for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate(
zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)):
# read in the SED file from the library
sed_file_name = os.path.join(sed_lib_dir, sed_names[s_dex])
sed = sims_photUtils.Sed()
sed.readSED_flambda(sed_file_name)
# find and apply normalizing flux
fnorm = sims_photUtils.getImsimFluxNorm(sed, mn)
sed.multiplyFluxNorm(fnorm)
# add internal dust
if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w):
ccm_w = np.copy(sed.wavelen)
a_x, b_x = sed.setupCCM_ab()
sed.addDust(a_x, b_x, A_v=av, R_v=rv)
# apply redshift
sed.redshiftSED(zz, dimming=True)
# flux, in Janskys, without Milky Way dust extinction
f_noMW = sed.calcFlux(lsst_bp_dict[bp])
# apply Milky Way dust
def process_instance_catalog(catalog_name, centroid_dir, bp_dict):
sed_dir = getPackageDir('sims_sed_library')
i_filter = -1
obshistid = -1
with open(catalog_name, 'r') as in_file:
for line in in_file:
params = line.strip().split()
if params[0] == 'filter':
i_filter = int(params[1])
if params[0] == 'obshistid':
obshistid = int(params[1])
if i_filter >= 0 and obshistid >= 0:
break
filter_name = 'ugrizy'[i_filter]
bp = bp_dict[filter_name]
objid_arr = []
flux_arr = []
redshift_arr = []
magnorm_arr = []
phot_params = PhotometricParameters(nexp=1, exptime=30.0, gain=1.0)
with open(catalog_name, 'r') as in_file:
for line in in_file:
params = line.strip().split()
if params[0] != 'object':
continue
sed_name = params[5]
obj_id = int(params[1])
magnorm = float(params[4])
redshift = float(params[6])
spec = Sed()
spec.readSED_flambda(os.path.join(sed_dir, sed_name))
fnorm = getImsimFluxNorm(spec, magnorm)
spec.multiplyFluxNorm(fnorm)
spec.redshiftSED(redshift, dimming=True)
adu = spec.calcADU(bp, photParams=phot_params)
objid_arr.append(obj_id)
flux_arr.append(adu)
redshift_arr.append(redshift)
magnorm_arr.append(magnorm)
centroid_files = os.listdir(centroid_dir)
phosim_objid_arr = []
phosim_flux_arr = []
x_arr = []
y_arr = []
chip_name_arr = []
dtype = np.dtype([('id', int), ('phot', float), ('x', float),
('y', float)])
for file_name in centroid_files:
if 'e_%d' % obshistid not in file_name or 'f%d' % i_filter not in file_name:
continue
name_params = file_name.split('_')
raft = name_params[5]
sensor = name_params[6]
chip_name = '%s:%s,%s %s:%s,%s' % (raft[0], raft[1], raft[2],
sensor[0], sensor[1], sensor[2])
full_name = os.path.join(centroid_dir, file_name)
data = np.genfromtxt(full_name, dtype=dtype, skip_header=1)
for line in data:
phosim_objid_arr.append(line['id'])
phosim_flux_arr.append(line['phot'])
x_arr.append(line['x'])
y_arr.append(line['y'])
chip_name_arr.append(chip_name)
objid_arr = np.array(objid_arr)
flux_arr = np.array(flux_arr)
redshift_arr = np.array(redshift_arr)
magnorm_arr = np.array(magnorm_arr)
phosim_objid_arr = np.array(phosim_objid_arr)
phosim_flux_arr = np.array(phosim_flux_arr)
x_arr = np.array(x_arr)
y_arr = np.array(y_arr)
chip_name_arr = np.array(chip_name_arr)
sorted_dex = np.argsort(objid_arr)
objid_arr = objid_arr[sorted_dex]
flux_arr = flux_arr[sorted_dex]
redshift_arr = redshift_arr[sorted_dex]
magnorm_arr = magnorm_arr[sorted_dex]
sorted_dex = np.argsort(phosim_objid_arr)
phosim_objid_arr = phosim_objid_arr[sorted_dex]
phosim_flux_arr = phosim_flux_arr[sorted_dex]
x_arr = x_arr[sorted_dex]
y_arr = y_arr[sorted_dex]
chip_name_arr = chip_name_arr[sorted_dex]
np.testing.assert_array_equal(phosim_objid_arr, objid_arr)
return (objid_arr, flux_arr, phosim_flux_arr, redshift_arr, magnorm_arr,
x_arr, y_arr, chip_name_arr, filter_name)
def load_dc2_hosts(catalog_version, agn_db, sed_dir):
# Load DC2 lens and host catalogs
dc2_reader = DC2Reader(catalog_version)
host_cat_filters = [
'ra < 75.', 'ra > 72.5', 'dec < -42.5', 'dec > -45.',
'redshift_true > 0.25'
]
host_galaxy_df = dc2_reader.load_galaxy_catalog(host_cat_filters)
# Load DC2 AGN
host_agn_query = 'ra > 72.5 and ra < 75. and dec > -45 and dec < -42.5 and redshift > 0.25'
host_agn_df = dc2_reader.load_agn_catalog(agn_db, host_agn_query)
# Load healpix ids so we can get SED information
healpix_ids_hosts = get_healpix_id(host_galaxy_df['ra'],
host_galaxy_df['dec'])
host_galaxy_df['healpix_id'] = healpix_ids_hosts
# To make SED loading quicker (we have to load an entire healpixel worth of SED
# info at a time) we will only take the four healpixels with the most galaxies.
# For more info see notebook matchingLensGalaxies.ipynb
healpix_query = 'healpix_id < 10203 or healpix_id == 10329 or healpix_id == 10452'
host_galaxy_df = host_galaxy_df.query(healpix_query).reset_index(drop=True)
# Join host and agn info
host_full_df = host_galaxy_df.join(host_agn_df.set_index('galaxy_id'),
on='galaxy_id',
rsuffix='_agn')
host_full_df = host_full_df.rename(
columns={
'magNorm': 'magNorm_agn',
'varParamStr': 'varParamStr_agn',
'redshift': 'redshift_agn'
})
# Pick out the host galaxies that have an AGN
# These will be our potential matches for lensed AGN
agn_host_full_df = host_full_df.query('ra_agn > -99.').reset_index(
drop=True)
# Host galaxies without an AGN are our pool of potential SNe host galaxies
sne_host_full_df = host_full_df.iloc[np.where(
np.isnan(host_full_df['M_i'].values))]
# Load i-band mags for AGN so we can match to OM10
lsst_bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
agn_sed = Sed()
agn_sed.readSED_flambda(
os.path.join(getPackageDir('SIMS_SED_LIBRARY'), 'agnSED',
'agn.spec.gz'))
mag_i_agn = []
for agn_data in agn_host_full_df[['magNorm_agn', 'redshift_true']].values:
agn_mag, agn_redshift = agn_data
if np.isnan(agn_mag):
mag_i_agn.append(np.nan)
continue
agn_copy = deepcopy(agn_sed)
flux_norm = getImsimFluxNorm(agn_copy, agn_mag)
agn_copy.multiplyFluxNorm(flux_norm)
agn_copy.redshiftSED(agn_redshift, dimming=True)
mag_i_agn.append(agn_copy.calcMag(lsst_bp_dict['i']))
agn_host_full_df['mag_i_agn'] = mag_i_agn
sed_df = pd.DataFrame([],
columns=[
'bulge_av',
'bulge_rv',
'bulge_sed',
'disk_av',
'disk_rv',
'disk_sed',
'galaxy_id',
])
for hpix in np.unique(host_full_df['healpix_id']):
# This file does not exist. And there are only 19 galaxies in this healpix anyway.
if hpix == 10572:
continue
f = h5py.File(os.path.join(sed_dir, 'sed_fit_%i.h5' % hpix))
sed_hpix_df = pd.DataFrame([])
band_list = ['u', 'g', 'r', 'i', 'z', 'y']
sed_names = f['sed_names'][()]
for key in list(f.keys()):
print(key, f[key].len())
# if (key.endswith('fluxes') or key.endswith('magnorm')):
if key.endswith('magnorm'):
key_data = f[key][()]
for i in range(6):
sed_hpix_df[str(key + '_' + band_list[i])] = key_data[i]
elif key in ['sed_names', 'ra', 'dec']:
continue
elif key.endswith('fluxes'):
continue
elif key.endswith('sed'):
sed_hpix_df[key] = [sed_names[idx] for idx in f[key][()]]
else:
sed_hpix_df[key] = f[key][()]
sed_df = pd.concat([sed_df, sed_hpix_df], sort=False)
sed_df = sed_df.reset_index(drop=True)
sed_df = sed_df.drop(columns=['redshift'])
agn_host_join = agn_host_full_df.join(sed_df.set_index('galaxy_id'),
on='galaxy_id')
sne_host_join = sne_host_full_df.join(sed_df.set_index('galaxy_id'),
on='galaxy_id')
# Clean up galaxies with bad magnorms
good_agn_rows = np.where((~np.isnan(agn_host_join['bulge_magnorm_u'])
& ~np.isinf(agn_host_join['bulge_magnorm_u'])
& ~np.isnan(agn_host_join['disk_magnorm_u'])
& ~np.isinf(agn_host_join['disk_magnorm_u'])))[0]
agn_host_final = agn_host_join.iloc[good_agn_rows].reset_index(drop=True)
good_sne_rows = np.where((~np.isnan(sne_host_join['bulge_magnorm_u'])
& ~np.isinf(sne_host_join['bulge_magnorm_u'])
& ~np.isnan(sne_host_join['disk_magnorm_u'])
& ~np.isinf(sne_host_join['disk_magnorm_u'])))[0]
sne_host_final = sne_host_join.iloc[good_sne_rows].reset_index(drop=True)
return agn_host_final, sne_host_final
idlrest = i_rest_dustless[i_star]
zdlrest = z_rest_dustless[i_star]
ydlrest = y_rest_dustless[i_star]
dustless = Sed()
sed = Sed()
rest_sed = Sed()
rest_sed_dustless = Sed()
full_sed_name = os.path.join(gal_sed_dir, sed_name)
sed.readSED_flambda(full_sed_name)
dustless.readSED_flambda(full_sed_name)
rest_sed.readSED_flambda(full_sed_name)
rest_sed_dustless.readSED_flambda(full_sed_name)
f_norm = getImsimFluxNorm(sed, mag_norm)
sed.multiplyFluxNorm(f_norm)
dustless.multiplyFluxNorm(f_norm)
f_norm = getImsimFluxNorm(rest_sed, rest_mag_norm)
rest_sed.multiplyFluxNorm(f_norm)
rest_sed_dustless.multiplyFluxNorm(f_norm)
a_x, b_x = sed.setupCCMab()
R_v = av/ebv
sed.addCCMDust(a_x, b_x, ebv=ebv, R_v=R_v)
rest_sed.addCCMDust(a_x, b_x, ebv=ebv, R_v=R_v)
sed.redshiftSED(full_redshift, dimming=True)
dustless.redshiftSED(full_redshift, dimming=True)
def loadSedsFromList(self, sedNameList, magNormList, \
internalAvList=None, galacticAvList=None, redshiftList=None):
"""
Load the Seds specified by sedNameList, applying the specified normalization,
extinction, and redshift.
@param [in] sedList is a list of file names containing Seds
@param [in] magNorm is the magnitude normalization
@param [in] internalAvList is an optional list of A(V) due to internal
dust
@param [in] galacticAvList is an optional list of A(V) due to
Milky Way dust
@param [in] redshiftList is an optional list of redshifts for the
input Sed
Seds are read in and stored to this object's internal list of Seds.
Note: if you constructed this SedList object without internalAvList,
you cannot load Seds with internalAvList now. Likewise for galacticAvlist
and redshiftList.
"""
if not self._initialized:
if internalAvList is not None:
self._internal_av_list = copy.deepcopy(list(internalAvList))
else:
self._internal_av_list = None
if galacticAvList is not None:
self._galactic_av_list = copy.deepcopy(list(galacticAvList))
else:
self._galactic_av_list = None
if redshiftList is not None:
self._redshift_list = copy.deepcopy(list(redshiftList))
else:
self._redshift_list = None
else:
if self._internal_av_list is None and internalAvList is not None:
raise RuntimeError("This SedList does not contain internalAvList")
elif self._internal_av_list is not None:
if internalAvList is None:
self._internal_av_list += [None] * len(sedNameList)
else:
self._internal_av_list += list(internalAvList)
if self._galactic_av_list is None and galacticAvList is not None:
raise RuntimeError("This SedList does not contain galacticAvList")
elif self._galactic_av_list is not None:
if galacticAvList is None:
self._galactic_av_list += [None] * len(sedNameList)
else:
self._galactic_av_list += list(galacticAvList)
if self._redshift_list is None and redshiftList is not None:
raise RuntimeError("This SedList does not contain redshiftList")
elif self._redshift_list is not None:
if redshiftList is None:
self._redshift_list += [None] * len(sedNameList)
else:
self._redshift_list += list(redshiftList)
temp_sed_list = []
for sedName, magNorm in zip(sedNameList, magNormList):
sed = Sed()
if sedName != "None":
if self._spec_map is not None:
sed.readSED_flambda(os.path.join(self._file_dir, self._spec_map[sedName]))
else:
sed.readSED_flambda(os.path.join(self._file_dir, sedName))
if self._normalizing_bandpass is not None:
fNorm = sed.calcFluxNorm(magNorm, self._normalizing_bandpass)
else:
fNorm = getImsimFluxNorm(sed, magNorm)
sed.multiplyFluxNorm(fNorm)
temp_sed_list.append(sed)
if internalAvList is not None:
self._av_int_wavelen, \
self._a_int, \
self._b_int = self.applyAv(temp_sed_list, internalAvList,
self._av_int_wavelen, self._a_int, self._b_int)
if redshiftList is not None:
self.applyRedshift(temp_sed_list, redshiftList)
if self._wavelen_match is not None:
for sedObj in temp_sed_list:
if sedObj.wavelen is not None:
sedObj.resampleSED(wavelen_match=self._wavelen_match)
if galacticAvList is not None:
self._av_gal_wavelen, \
self._a_gal, \
self._b_gal = self.applyAv(temp_sed_list, galacticAvList,
self._av_gal_wavelen, self._a_gal, self._b_gal)
self._sed_list += temp_sed_list
self._initialized = True
t_id = twinkles_data['twinklesId'][idx]
sed_name = twinkles_data['lens_sed'][idx]
i_magnorm = twinkles_data['sed_magNorm'][idx][3]
apmag_i = twinkles_data['APMAG_I'][idx]
av = twinkles_data['lens_av'][idx]
rv = twinkles_data['lens_rv'][idx]
#valid = np.where(gal_to_twink['twinkles_id']==t_id)[0]
#g_id = gal_to_twink['galaxy_id'][valid]
#valid = np.where(galaxy_id==g_id)[0]
zz = twinkles_data['ZLENS'][idx]
spec = sims_photUtils.Sed()
full_file_name = os.path.join(sed_dir, sed_name)
spec.readSED_flambda(full_file_name)
fnorm = sims_photUtils.getImsimFluxNorm(spec, i_magnorm)
spec.multiplyFluxNorm(fnorm)
if dust_wav is None or not np.array_equal(spec.wavelen, dust_wav):
dust_wav = np.copy(spec.wavelen)
a_x, b_x = spec.setupCCM_ab()
spec.addDust(a_x, b_x, A_v=av, R_v=rv)
spec.redshiftSED(zz, dimming=True)
i_mag = spec.calcMag(bp_dict['i'])
d_mag = i_mag-apmag_i
abs_d_mag = np.abs(i_mag-apmag_i)
if abs_d_mag > max_dmag:
print('d_mag %e: is %e shld %e; %d' %
(d_mag, i_mag, apmag_i, twinkles_data['LENSID'][idx]))
print('av %e rv %e zz %e sed %s magnorm %e AP %e' %
(av, rv, zz, sed_name,i_magnorm,apmag_i))
max_dmag = abs_d_mag
def validate_chunk(data_in,
in_dir, healpix,
read_lock, write_lock, output_dict):
galaxy_id = data_in['galaxy_id']
redshift = data_in['redshift']
mag_in = {}
for bp in 'ugrizy':
mag_in[bp] = data_in['mag_true_%s_lsst' % bp]
eps = 1.0e-20
disk_to_bulge = {}
for bp in 'ugrizy':
disk_name = 'LSST_filters/diskLuminositiesStellar:'
disk_name += 'LSST_%s:observed:dustAtlas' % bp
bulge_name = 'LSST_filters/spheroidLuminositiesStellar:'
bulge_name += 'LSST_%s:observed:dustAtlas' % bp
denom = np.where(data_in[bulge_name]>0.0, data_in[bulge_name], eps)
disk_to_bulge[bp] = data_in[disk_name]/denom
assert np.isfinite(disk_to_bulge[bp]).sum()==len(disk_to_bulge[bp])
sed_dir = os.environ['SIMS_SED_LIBRARY_DIR']
tot_bp_dict = photUtils.BandpassDict.loadTotalBandpassesFromFiles()
fit_file = os.path.join(in_dir, 'sed_fit_%d.h5' % healpix)
assert os.path.isfile(fit_file)
with read_lock:
with h5py.File(fit_file, 'r') as in_file:
sed_names = in_file['sed_names'][()]
galaxy_id_fit = in_file['galaxy_id'][()]
valid = np.in1d(galaxy_id_fit, galaxy_id)
galaxy_id_fit = galaxy_id_fit[valid]
np.testing.assert_array_equal(galaxy_id_fit, galaxy_id)
disk_magnorm = in_file['disk_magnorm'][()][:,valid]
disk_av = in_file['disk_av'][()][valid]
disk_rv = in_file['disk_rv'][()][valid]
disk_sed = in_file['disk_sed'][()][valid]
bulge_magnorm = in_file['bulge_magnorm'][()][:,valid]
bulge_av = in_file['bulge_av'][()][valid]
bulge_rv = in_file['bulge_rv'][()][valid]
bulge_sed = in_file['bulge_sed'][()][valid]
sed_names = [name.decode() for name in sed_names]
local_worst = {}
local_worst_ratio = {}
ccm_w = None
dummy_sed = photUtils.Sed()
for i_obj in range(len(disk_av)):
for i_bp, bp in enumerate('ugrizy'):
d_sed = photUtils.Sed()
fname = os.path.join(sed_dir, sed_names[disk_sed[i_obj]])
d_sed.readSED_flambda(fname)
fnorm = photUtils.getImsimFluxNorm(d_sed,
disk_magnorm[i_bp][i_obj])
d_sed.multiplyFluxNorm(fnorm)
if ccm_w is None or not np.array_equal(d_sed.wavelen, ccm_w):
ccm_w = np.copy(d_sed.wavelen)
ax, bx = d_sed.setupCCM_ab()
d_sed.addDust(ax, bx, R_v=disk_rv[i_obj], A_v=disk_av[i_obj])
d_sed.redshiftSED(redshift[i_obj], dimming=True)
d_flux = d_sed.calcFlux(tot_bp_dict[bp])
b_sed = photUtils.Sed()
fname = os.path.join(sed_dir, sed_names[bulge_sed[i_obj]])
b_sed.readSED_flambda(fname)
fnorm = photUtils.getImsimFluxNorm(b_sed,
bulge_magnorm[i_bp][i_obj])
b_sed.multiplyFluxNorm(fnorm)
if ccm_w is None or not np.array_equal(b_sed.wavelen, ccm_w):
ccm_w = np.copy(b_sed.wavelen)
ax, bx = b_sed.setupCCM_ab()
b_sed.addDust(ax, bx, R_v=bulge_rv[i_obj], A_v=bulge_av[i_obj])
b_sed.redshiftSED(redshift[i_obj], dimming=True)
b_flux = b_sed.calcFlux(tot_bp_dict[bp])
flux_ratio = d_flux/max(b_flux, eps)
if flux_ratio<1.0e3 or disk_to_bulge[bp][i_obj]<1.0e3:
delta_ratio = np.abs(flux_ratio-disk_to_bulge[bp][i_obj])
if disk_to_bulge[bp][i_obj]>eps:
delta_ratio /= disk_to_bulge[bp][i_obj]
if bp not in local_worst_ratio or delta_ratio>local_worst_ratio[bp]:
local_worst_ratio[bp] = delta_ratio
tot_flux = b_flux + d_flux
true_flux = dummy_sed.fluxFromMag(mag_in[bp][i_obj])
delta_flux = np.abs(1.0-tot_flux/true_flux)
if bp not in local_worst or delta_flux>local_worst[bp]:
local_worst[bp] = delta_flux
with write_lock:
for bp in 'ugrizy':
if bp not in output_dict or local_worst[bp]>output_dict[bp]:
output_dict[bp] = local_worst[bp]
ratio_name = '%s_ratio' % bp
if (ratio_name not in output_dict
or local_worst_ratio[bp]>output_dict[ratio_name]):
output_dict[ratio_name] = local_worst_ratio[bp]
def sources_from_file(file_name, obs_md, phot_params, numRows=None):
"""
Read in an InstanceCatalog and extract all of the astrophysical
sources from it
Parameters
----------
file_name: str
The name of the InstanceCatalog
obs_md: ObservationMetaData
The ObservationMetaData characterizing the pointing
phot_params: PhotometricParameters
The PhotometricParameters characterizing this telescope
numRows: int (optional)
The number of rows of the InstanceCatalog to read in (including the
header)
Returns
-------
gs_obj_arr: numpy array
Contains the GalSimCelestialObjects for all of the
astrophysical sources in this InstanceCatalog
out_obj_dict: dict
Keyed on the names of the detectors in the LSST camera.
The values are numpy arrays of GalSimCelestialObjects
that should be simulated for that detector, including
objects that are near the edge of the chip or
just bright (in which case, they might still illuminate
the detector).
"""
camera = get_obs_lsstSim_camera()
num_objects = 0
ct_rows = 0
with fopen(file_name, mode='rt') as input_file:
for line in input_file:
ct_rows += 1
params = line.strip().split()
if params[0] == 'object':
num_objects += 1
if numRows is not None and ct_rows >= numRows:
break
# RA, Dec in the coordinate system expected by PhoSim
ra_phosim = np.zeros(num_objects, dtype=float)
dec_phosim = np.zeros(num_objects, dtype=float)
sed_name = [None] * num_objects
mag_norm = 55.0 * np.ones(num_objects, dtype=float)
gamma1 = np.zeros(num_objects, dtype=float)
gamma2 = np.zeros(num_objects, dtype=float)
kappa = np.zeros(num_objects, dtype=float)
internal_av = np.zeros(num_objects, dtype=float)
internal_rv = np.zeros(num_objects, dtype=float)
galactic_av = np.zeros(num_objects, dtype=float)
galactic_rv = np.zeros(num_objects, dtype=float)
semi_major_arcsec = np.zeros(num_objects, dtype=float)
semi_minor_arcsec = np.zeros(num_objects, dtype=float)
position_angle_degrees = np.zeros(num_objects, dtype=float)
sersic_index = np.zeros(num_objects, dtype=float)
npoints = np.zeros(num_objects, dtype=int)
redshift = np.zeros(num_objects, dtype=float)
unique_id = np.zeros(num_objects, dtype=int)
object_type = np.zeros(num_objects, dtype=int)
i_obj = -1
with fopen(file_name, mode='rt') as input_file:
for line in input_file:
params = line.strip().split()
if params[0] != 'object':
continue
i_obj += 1
if numRows is not None and i_obj >= num_objects:
break
unique_id[i_obj] = int(params[1])
ra_phosim[i_obj] = float(params[2])
dec_phosim[i_obj] = float(params[3])
mag_norm[i_obj] = float(params[4])
sed_name[i_obj] = params[5]
redshift[i_obj] = float(params[6])
gamma1[i_obj] = float(params[7])
gamma2[i_obj] = float(params[8])
kappa[i_obj] = float(params[9])
if params[12].lower() == 'point':
object_type[i_obj] = _POINT_SOURCE
i_gal_dust_model = 14
if params[13].lower() != 'none':
i_gal_dust_model = 16
internal_av[i_obj] = float(params[14])
internal_rv[i_obj] = float(params[15])
if params[i_gal_dust_model].lower() != 'none':
galactic_av[i_obj] = float(params[i_gal_dust_model + 1])
galactic_rv[i_obj] = float(params[i_gal_dust_model + 2])
elif params[12].lower() == 'sersic2d':
object_type[i_obj] = _SERSIC_2D
semi_major_arcsec[i_obj] = float(params[13])
semi_minor_arcsec[i_obj] = float(params[14])
position_angle_degrees[i_obj] = float(params[15])
sersic_index[i_obj] = float(params[16])
i_gal_dust_model = 18
if params[17].lower() != 'none':
i_gal_dust_model = 20
internal_av[i_obj] = float(params[18])
internal_rv[i_obj] = float(params[19])
if params[i_gal_dust_model].lower() != 'none':
galactic_av[i_obj] = float(params[i_gal_dust_model + 1])
galactic_rv[i_obj] = float(params[i_gal_dust_model + 2])
elif params[12].lower() == 'knots':
object_type[i_obj] = _RANDOM_WALK
semi_major_arcsec[i_obj] = float(params[13])
semi_minor_arcsec[i_obj] = float(params[14])
position_angle_degrees[i_obj] = float(params[15])
npoints[i_obj] = int(params[16])
i_gal_dust_model = 18
if params[17].lower() != 'none':
i_gal_dust_model = 20
internal_av[i_obj] = float(params[18])
internal_rv[i_obj] = float(params[19])
if params[i_gal_dust_model].lower() != 'none':
galactic_av[i_obj] = float(params[i_gal_dust_model + 1])
galactic_rv[i_obj] = float(params[i_gal_dust_model + 2])
else:
raise RuntimeError("Do not know how to handle "
"object type: %s" % params[12])
ra_appGeo, dec_appGeo = PhoSimAstrometryBase._appGeoFromPhoSim(
np.radians(ra_phosim), np.radians(dec_phosim), obs_md)
(ra_obs_rad, dec_obs_rad) = _observedFromAppGeo(ra_appGeo,
dec_appGeo,
obs_metadata=obs_md,
includeRefraction=True)
semi_major_radians = radiansFromArcsec(semi_major_arcsec)
semi_minor_radians = radiansFromArcsec(semi_minor_arcsec)
position_angle_radians = np.radians(position_angle_degrees)
x_pupil, y_pupil = _pupilCoordsFromObserved(ra_obs_rad, dec_obs_rad,
obs_md)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
sed_dir = lsstUtils.getPackageDir('sims_sed_library')
object_is_valid = np.array([True] * num_objects)
invalid_objects = np.where(
np.logical_or(
np.logical_or(
mag_norm > 50.0,
np.logical_and(galactic_av == 0.0, galactic_rv == 0.0)),
np.logical_or(
np.logical_and(object_type == _SERSIC_2D,
semi_major_arcsec < semi_minor_arcsec),
np.logical_and(object_type == _RANDOM_WALK, npoints <= 0))))
object_is_valid[invalid_objects] = False
if len(invalid_objects[0]) > 0:
message = "\nOmitted %d suspicious objects from " % len(
invalid_objects[0])
message += "the instance catalog:\n"
n_bad_mag_norm = len(np.where(mag_norm > 50.0)[0])
message += " %d had mag_norm > 50.0\n" % n_bad_mag_norm
n_bad_av = len(
np.where(np.logical_and(galactic_av == 0.0,
galactic_rv == 0.0))[0])
message += " %d had galactic_Av == galactic_Rv == 0\n" % n_bad_av
n_bad_axes = len(
np.where(
np.logical_and(object_type == _SERSIC_2D,
semi_major_arcsec < semi_minor_arcsec))[0])
message += " %d had semi_major_axis < semi_minor_axis\n" % n_bad_axes
n_bad_knots = len(
np.where(np.logical_and(object_type == _RANDOM_WALK,
npoints <= 0))[0])
message += " %d had n_points <= 0 \n" % n_bad_knots
warnings.warn(message)
wav_int = None
wav_gal = None
gs_object_arr = []
for i_obj in range(num_objects):
if not object_is_valid[i_obj]:
continue
if object_type[i_obj] == _POINT_SOURCE:
gs_type = 'pointSource'
elif object_type[i_obj] == _SERSIC_2D:
gs_type = 'sersic'
elif object_type[i_obj] == _RANDOM_WALK:
gs_type = 'RandomWalk'
# load the SED
sed_obj = Sed()
sed_obj.readSED_flambda(os.path.join(sed_dir, sed_name[i_obj]))
fnorm = getImsimFluxNorm(sed_obj, mag_norm[i_obj])
sed_obj.multiplyFluxNorm(fnorm)
if internal_av[i_obj] != 0.0:
if wav_int is None or not np.array_equal(sed_obj.wavelen, wav_int):
a_int, b_int = sed_obj.setupCCMab()
wav_int = copy.deepcopy(sed_obj.wavelen)
sed_obj.addCCMDust(a_int,
b_int,
A_v=internal_av[i_obj],
R_v=internal_rv[i_obj])
if redshift[i_obj] != 0.0:
sed_obj.redshiftSED(redshift[i_obj], dimming=True)
sed_obj.resampleSED(wavelen_match=bp_dict.wavelenMatch)
if galactic_av[i_obj] != 0.0:
if wav_gal is None or not np.array_equal(sed_obj.wavelen, wav_gal):
a_g, b_g = sed_obj.setupCCMab()
wav_gal = copy.deepcopy(sed_obj.wavelen)
sed_obj.addCCMDust(a_g,
b_g,
A_v=galactic_av[i_obj],
R_v=galactic_rv[i_obj])
gs_object = GalSimCelestialObject(gs_type,
x_pupil[i_obj],
y_pupil[i_obj],
semi_major_radians[i_obj],
semi_minor_radians[i_obj],
semi_major_radians[i_obj],
position_angle_radians[i_obj],
sersic_index[i_obj],
sed_obj,
bp_dict,
phot_params,
npoints[i_obj],
gamma1=gamma1[i_obj],
gamma2=gamma2[i_obj],
kappa=kappa[i_obj],
uniqueId=unique_id[i_obj])
gs_object_arr.append(gs_object)
gs_object_arr = np.array(gs_object_arr)
# how close to the edge of the detector a source has
# to be before we will just simulate it anyway
pix_tol = 50.0
# any source brighter than this will be considered
# so bright that it should be simulated for all
# detectors, just in case light scatters onto them.
max_mag = 16.0
# down-select mag_norm, x_pupil, and y_pupil
# to only contain those objects that were
# deemed to be valid above
valid = np.where(object_is_valid)
mag_norm = mag_norm[valid]
x_pupil = x_pupil[valid]
y_pupil = y_pupil[valid]
assert len(mag_norm) == len(gs_object_arr)
assert len(x_pupil) == len(gs_object_arr)
assert len(y_pupil) == len(gs_object_arr)
out_obj_dict = {}
for det in lsst_camera():
chip_name = det.getName()
pixel_corners = getCornerPixels(chip_name, lsst_camera())
x_min = pixel_corners[0][0]
x_max = pixel_corners[2][0]
y_min = pixel_corners[0][1]
y_max = pixel_corners[3][1]
xpix, ypix = pixelCoordsFromPupilCoords(x_pupil,
y_pupil,
chipName=chip_name,
camera=lsst_camera())
on_chip = np.where(
np.logical_or(
mag_norm < max_mag,
np.logical_and(
xpix > x_min - pix_tol,
np.logical_and(
xpix < x_max + pix_tol,
np.logical_and(ypix > y_min - pix_tol,
ypix < y_max + pix_tol)))))
out_obj_dict[chip_name] = gs_object_arr[on_chip]
return gs_object_arr, out_obj_dict
def _process_chunk(db_lock, log_lock, sema, sed_fit_name, cosmoDC2_data,
first_gal, self_dict, bad_gals):
"""
Do all chunk-specific work: compute table contents for a
collection of galaxies and write to db
Parameters
----------
db_lock Used to avoid conflicts writing to sqlite output
log_lock Used to avoid conflicts writing to per-healpixel log
sema A semaphore. Release when done
sed_fit_name File where sed fits for this healpixel are
cosmoDC2_data Values from cosmoDC2 for this healpixel, keyed by
column name
first_gal index of first galaxy in our chunk (in sed fit list)
self_dict Random useful values stored in GalaxyTruthWriter
bad_gals List of galaxy ids, monotone increasing, to be
skipped
"""
dry = self_dict['dry']
chunk_size = self_dict['chunk_size']
dbfile = self_dict['dbfile']
logfile = self_dict['logfile']
if dry:
_logit(
log_lock, logfile,
'_process_chunk invoke for first_gal {}, chunk size {}'.format(
first_gal, chunk_size))
if sema is None:
return
sema.release()
#exit(0)
return
lsst_bp_dict = self_dict['lsst_bp_dict']
galaxy_ids = []
ra = []
dec = []
redshift = []
ebv_vals = None
ebv_vals_init = False # does this belong somewhere else?
ccm_w = None
total_gals = self_dict['total_gals']
chunk_start = first_gal
chunk_end = min(first_gal + chunk_size, total_gals)
with h5py.File(sed_fit_name, 'r') as sed_fit_file:
sed_names = sed_fit_file['sed_names'][()]
sed_names = [s.decode() for s in sed_names] # becse stored as bytes
gals_this_chunk = chunk_end - chunk_start
subset = slice(chunk_start, chunk_end)
galaxy_ids = sed_fit_file['galaxy_id'][()][subset]
to_log = 'Start with galaxy #{}, id={}\n# galaxies for _process_chunk: {}\n'.format(
first_gal, galaxy_ids[0], len(galaxy_ids))
_logit(log_lock, logfile, to_log)
# get the cross-match between the sed fit and cosmoDC2
cosmo_len = len(cosmoDC2_data['galaxy_id'])
crossmatch_dex = np.searchsorted(cosmoDC2_data['galaxy_id'],
galaxy_ids)
np.testing.assert_array_equal(
galaxy_ids, cosmoDC2_data['galaxy_id'][crossmatch_dex])
ra = sed_fit_file['ra'][()][subset]
dec = sed_fit_file['dec'][()][subset]
np.testing.assert_array_equal(ra, cosmoDC2_data['ra'][crossmatch_dex])
np.testing.assert_array_equal(dec,
cosmoDC2_data['dec'][crossmatch_dex])
good_ixes = _good_indices(galaxy_ids.tolist(), bad_gals[0])
if (len(good_ixes) == 0):
if sema is not None:
sema.release()
return
else:
_logit(
log_lock, logfile,
'Found {} good indices for chunk starting with {}\n'.format(
len(good_ixes), chunk_start))
flux_by_band_MW = {}
flux_by_band_noMW = {}
# Calculate E(B-V) for dust extinction in Milky Way along relevant
# lines of sight
band_print = "Processing band {}, first gal {}, time {}\n"
if not ebv_vals_init:
equatorial_coords = np.array([np.radians(ra), np.radians(dec)])
ebv_model = EBVbase()
ebv_vals = ebv_model.calculateEbv(
equatorialCoordinates=equatorial_coords, interp=True)
ebv_vals_init = True
for i_bp, bp in enumerate('ugrizy'):
if (i_bp == 0 or i_bp == 5):
_logit(log_lock, logfile,
band_print.format(bp, first_gal, dt.now()))
fluxes_noMW = {}
fluxes = {}
for component in ['disk', 'bulge']:
fluxes_noMW[component] = np.zeros(gals_this_chunk, dtype=float)
fluxes[component] = np.zeros(gals_this_chunk, dtype=float)
for component in ['disk', 'bulge']:
#print(" Processing component ", component)
sed_arr = sed_fit_file['%s_sed' % component][()][subset]
av_arr = sed_fit_file['%s_av' % component][()][subset]
rv_arr = sed_fit_file['%s_rv' % component][()][subset]
mn_arr = sed_fit_file['%s_magnorm' %
component][()][i_bp, :][subset]
z_arr = cosmoDC2_data['redshift'][crossmatch_dex]
gii = 0
done = False
for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate(
zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)):
if done: break
while good_ixes[gii] < i_gal:
gii += 1
if gii == len(good_ixes): # ran out of good ones
done = True
break
if done: break
if good_ixes[gii] > i_gal: # skipped over it; it's bad
continue
# Leave space for it in the arrays, but values
# for all the fluxes will be left at 0
# read in the SED file from the library
sed_file_name = os.path.join(self_dict['sed_lib_dir'],
sed_names[s_dex])
sed = sims_photUtils.Sed()
sed.readSED_flambda(sed_file_name)
# find and apply normalizing flux
fnorm = sims_photUtils.getImsimFluxNorm(sed, mn)
sed.multiplyFluxNorm(fnorm)
# add internal dust
if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w):
ccm_w = np.copy(sed.wavelen)
a_x, b_x = sed.setupCCM_ab()
sed.addDust(a_x, b_x, A_v=av, R_v=rv)
# apply redshift
sed.redshiftSED(zz, dimming=True)
# flux, in Janskys, without Milky Way dust extinction
f_noMW = sed.calcFlux(lsst_bp_dict[bp])
# apply Milky Way dust
# (cannot reuse a_x, b_x because wavelength grid changed
# when we called redshiftSED)
a_x_mw, b_x_mw = sed.setupCCM_ab()
sed.addDust(a_x_mw, b_x_mw, R_v=3.1, ebv=ebv)
f_MW = sed.calcFlux(lsst_bp_dict[bp])
fluxes_noMW[component][i_gal] = f_noMW
fluxes[component][i_gal] = f_MW
if (component == 'disk') and (bp == 'r'):
redshift = z_arr
# Sum components and convert to nanojansky
total_fluxes = (fluxes_noMW['disk'] + fluxes_noMW['bulge']) * 10**9
total_fluxes_MW = (fluxes['disk'] + fluxes['bulge']) * 10**9
dummy_sed = sims_photUtils.Sed()
# add magnification due to weak lensing
kappa = cosmoDC2_data['convergence'][crossmatch_dex]
gamma_sq = (cosmoDC2_data['shear_1'][crossmatch_dex]**2 +
cosmoDC2_data['shear_2'][crossmatch_dex]**2)
magnification = 1.0 / ((1.0 - kappa)**2 - gamma_sq)
magnified_fluxes = magnification * total_fluxes
magnified_fluxes_MW = magnification * total_fluxes_MW
flux_by_band_noMW[bp] = magnified_fluxes
flux_by_band_MW[bp] = magnified_fluxes_MW
# Open connection to sqlite db and write
#print('Time before db write is {}, first gal={}'.format(dt.now(), first_gal))
#sys.stdout.flush()
if not db_lock.acquire(timeout=120.0):
_logit(log_lock, logfile, "Failed to acquire db lock, first gal=",
first_gal)
if sema is None:
return
sema.release()
exit(1)
try:
_write_sqlite(dbfile, galaxy_ids, ra, dec, redshift, flux_by_band_MW,
flux_by_band_noMW, good_ixes)
db_lock.release()
if sema is not None:
sema.release()
_logit(
log_lock, logfile,
'Time after db write: {}, first_gal={}\n'.format(
dt.now(), first_gal))
exit(0)
except Exception as ex:
db_lock.release()
if sema is not None:
sema.release()
raise (ex)
def validate_sne(cat_dir, obsid, fov_deg=2.1,
scatter_file=None, vanishing_file=None):
"""
Parameters
----------
cat_dir is the parent dir of $obsid
obsid is the obsHistID of the pointing (an int)
fov_deg is the radius of the field of view in degrees
"""
project_dir = os.path.join('/global/projecta/projectdirs',
'lsst/groups/SSim/DC2/cosmoDC2_v1.1.4')
sne_db_name = os.path.join(project_dir, 'sne_cosmoDC2_v1.1.4_MS_DDF.db')
if not os.path.isfile(sne_db_name):
raise RuntimeError("\n%s\nis not a file" % sne_db_name)
instcat_dir = os.path.join(cat_dir, '%.8d' % obsid)
if not os.path.isdir(instcat_dir):
raise RuntimeError("\n%s\nis not a dir" % instcat_dir)
sne_name = os.path.join(instcat_dir, 'sne_cat_%d.txt.gz' % obsid)
if not os.path.isfile(sne_name):
raise RuntimeError("\n%s\nis not a file" % sne_name)
phosim_name = os.path.join(instcat_dir, 'phosim_cat_%d.txt' % obsid)
if not os.path.isfile(phosim_name):
raise RuntimeError("\n%s\nis not a file" % phosim_name)
sed_dir = os.path.join(instcat_dir, "Dynamic")
if not os.path.isdir(sed_dir):
raise RuntimeError("\n%s\nis not a dir" % sed_dir)
opsim_db = os.path.join("/global/projecta/projectdirs",
"lsst/groups/SSim/DC2",
"minion_1016_desc_dithered_v4_sfd.db")
if not os.path.isfile(opsim_db):
raise RuntimeError("\n%s\nis not a file" % opsim_db)
band_from_int = 'ugrizy'
bandpass = None
with open(phosim_name, 'r') as in_file:
for line in in_file:
params = line.strip().split()
if params[0] == 'filter':
bandpass = band_from_int[int(params[1])]
break
if bandpass is None:
raise RuntimeError("Failed to read in bandpass")
bp_dict = photUtils.BandpassDict.loadTotalBandpassesFromFiles()
with sqlite3.connect(opsim_db) as conn:
c = conn.cursor()
r = c.execute("SELECT descDitheredRA, descDitheredDec, expMJD FROM Summary "
"WHERE obsHistID==%d" % obsid).fetchall()
pointing_ra = np.degrees(r[0][0])
pointing_dec = np.degrees(r[0][1])
expmjd = float(r[0][2])
with sqlite3.connect(sne_db_name) as conn:
c = conn.cursor()
query = "SELECT snid_in, snra_in, sndec_in, "
query += "c_in, mB, t0_in, x0_in, x1_in, z_in "
query += "FROM sne_params WHERE ("
where_clause = None
half_space = htm.halfSpaceFromRaDec(pointing_ra, pointing_dec, fov_deg)
bounds = half_space.findAllTrixels(6)
for bound in bounds:
if where_clause is not None:
where_clause += " OR ("
else:
where_clause = "("
if bound[0] == bound[1]:
where_clause += "htmid_level_6==%d" % bound[0]
else:
where_clause += "htmid_level_6>=%d AND htmid_level_6<=%d" % (bound[0],bound[1])
where_clause += ")"
query += where_clause+")"
sn_params = c.execute(query).fetchall()
sn_ra = np.array([sn[1] for sn in sn_params])
sn_dec = np.array([sn[2] for sn in sn_params])
sn_d = angularSeparation(pointing_ra, pointing_dec,
sn_ra, sn_dec)
valid = np.where(sn_d<=fov_deg)
sn_ra_dec = {}
sn_param_dict = {}
for i_sn in valid[0]:
sn = sn_params[i_sn]
sn_param_dict[sn[0]] = sn[3:]
sn_ra_dec[sn[0]] = sn[1:3]
sne_in_instcat = set()
d_mag_max = -1.0
with gzip.open(sne_name, 'rb') as sne_cat_file:
for sne_line in sne_cat_file:
instcat_params = sne_line.strip().split(b' ')
sne_id = instcat_params[1].decode('utf-8')
if sne_id not in sn_param_dict:
try:
sne_id_int = int(sne_id)
assert sne_id_int<1.5e10
except (ValueError, AssertionError):
raise RuntimeError("\n%s\nnot in SNe db" % sne_id)
sne_in_instcat.add(sne_id)
control_params = sn_param_dict[sne_id]
sed_name = os.path.join(instcat_dir, instcat_params[5].decode('utf-8'))
if not os.path.isfile(sed_name):
raise RuntimeError("\n%s\nis not a file" % sed_name)
sed = photUtils.Sed()
sed.readSED_flambda(sed_name, cache_sed=False)
fnorm = photUtils.getImsimFluxNorm(sed, float(instcat_params[4]))
sed.multiplyFluxNorm(fnorm)
sed.redshiftSED(float(instcat_params[6]), dimming=True)
a_x, b_x = sed.setupCCM_ab()
A_v = float(instcat_params[15])
R_v = float(instcat_params[16])
EBV = A_v/R_v
sed.addDust(a_x, b_x, A_v=A_v, R_v=R_v)
sn_object = SNObject()
sn_object.set_MWebv(EBV)
sn_object.set(c=control_params[0], x1=control_params[4],
x0=control_params[3], t0=control_params[2],
z=control_params[5])
sn_mag = sn_object.catsimBandMag(bp_dict[bandpass], expmjd)
instcat_flux = sed.calcFlux(bp_dict[bandpass])
instcat_mag = sed.magFromFlux(instcat_flux)
d_mag = (sn_mag-instcat_mag)
if not np.isfinite(d_mag):
if np.isfinite(sn_mag) or np.isfinite(instcat_mag):
msg = '%s\ngave magnitudes %e %e (diff %e)' % (sne_id,
sn_mag,
instcat_mag,
d_mag)
print(msg)
if scatter_file is not None:
scatter_file.write("%e %e %e\n" % (sn_mag, instcat_mag, d_mag))
d_mag = np.abs(d_mag)
if d_mag>d_mag_max:
d_mag_max = d_mag
#print('dmag %e -- %e (%e)' %
# (d_mag, instcat_mag,
# control_params[5]-float(instcat_params[6])))
msg = '\n%s\n' % sne_name
ct_missing = 0
for sn_id in sn_param_dict:
if sn_id in sne_in_instcat:
continue
control_params = sn_param_dict[sn_id]
sn_object = SNObject()
sn_object.set_MWebv(0.0)
cc = control_params[0]
x1 = control_params[4]
x0 = control_params[3]
t0 = control_params[2]
zz = control_params[5]
sn_object.set(c=cc, x1=x1, x0=x0, t0=t0, z=zz)
sn_mag = sn_object.catsimBandMag(bp_dict[bandpass], expmjd)
if vanishing_file is not None and np.isfinite(sn_mag):
vanishing_file.write('%d %s %s %e ' % (obsid, bandpass, sn_id, sn_mag))
vanishing_file.write('%e %e %e %.4f %e %.4f %e\n' %
(cc,x0,x1,t0,zz,expmjd,expmjd-t0))
def write_sprinkled_lc(out_file_name, total_obs_md,
pointing_dir, opsim_db_name,
ra_colname='descDitheredRA',
dec_colname='descDitheredDec',
rottel_colname = 'descDitheredRotTelPos',
sql_file_name=None,
bp_dict=None):
"""
Create database of light curves
Note: this is still under development. It has not yet been
used for a production-level truth catalog
Parameters
----------
out_file_name is the name of the sqlite file to be written
total_obs_md is an ObservationMetaData covering the whole
survey area
pointing_dir contains a series of files that are two columns: obshistid, mjd.
The files must each have 'visits' in their name. These specify the pointings
for which we are assembling data. See:
https://github.com/LSSTDESC/DC2_Repo/tree/master/data/Run1.1
for an example.
opsim_db_name is the name of the OpSim database to be queried for pointings
ra_colname is the column used for RA of the pointing (default:
descDitheredRA)
dec_colname is the column used for the Dec of the pointing (default:
descDitheredDec)
rottel_colname is the column used for the rotTelPos of the pointing
(default: desckDitheredRotTelPos')
sql_file_name is the name of the parameter database produced by
write_sprinkled_param_db to be used
bp_dict is a BandpassDict of the telescope filters to be used
Returns
-------
None
Writes out a database to out_file_name. The tables of this database and
their columns are:
light_curves:
- uniqueId -- an int unique to all objects
- obshistid -- an int unique to all pointings
- mag -- the magnitude observed for this object at that pointing
obs_metadata:
- obshistid -- an int unique to all pointings
- mjd -- the date of the pointing
- filter -- an int corresponding to the telescope filter (0==u, 1==g..)
variables_and_transients:
- uniqueId -- an int unique to all objects
- galaxy_id -- an int indicating the host galaxy
- ra -- in degrees
- dec -- in degrees
- agn -- ==1 if object is an AGN
- sn -- ==1 if object is a supernova
"""
t0_master = time.time()
if not os.path.isfile(sql_file_name):
raise RuntimeError('%s does not exist' % sql_file_name)
sn_simulator = SneSimulator(bp_dict)
sed_dir = os.environ['SIMS_SED_LIBRARY_DIR']
create_sprinkled_sql_file(out_file_name)
t_start = time.time()
# get data about the pointings being simulated
(htmid_dict,
mjd_dict,
filter_dict,
obsmd_dict) = get_pointing_htmid(pointing_dir, opsim_db_name,
ra_colname=ra_colname,
dec_colname=dec_colname)
t_htmid_dict = time.time()-t_start
bp_to_int = {'u':0, 'g':1, 'r':2, 'i':3, 'z':4, 'y':5}
# put the data about the pointings in the obs_metadata table
with sqlite3.connect(out_file_name) as conn:
cursor = conn.cursor()
values = ((int(obs), mjd_dict[obs], bp_to_int[filter_dict[obs]])
for obs in mjd_dict)
cursor.executemany('''INSERT INTO obs_metadata VALUES (?,?,?)''', values)
cursor.execute('''CREATE INDEX obs_filter
ON obs_metadata (obshistid, filter)''')
conn.commit()
print('\ngot htmid_dict -- %d in %e seconds' % (len(htmid_dict), t_htmid_dict))
db = DBObject(sql_file_name, driver='sqlite')
# get a list of htmid corresponding to trixels in which
# variables and transients can be found
query = 'SELECT DISTINCT htmid FROM zpoint WHERE is_agn=1 OR is_sn=1'
dtype = np.dtype([('htmid', int)])
results = db.execute_arbitrary(query, dtype=dtype)
object_htmid = results['htmid']
agn_dtype = np.dtype([('uniqueId', int), ('galaxy_id', int),
('ra', float), ('dec', float),
('redshift', float), ('sed', str, 500),
('magnorm', float), ('varParamStr', str, 500),
('is_sprinkled', int)])
agn_base_query = 'SELECT uniqueId, galaxy_id, '
agn_base_query += 'raJ2000, decJ2000, '
agn_base_query += 'redshift, sedFilepath, '
agn_base_query += 'magNorm, varParamStr, is_sprinkled '
agn_base_query += 'FROM zpoint WHERE is_agn=1 '
sn_dtype = np.dtype([('uniqueId', int), ('galaxy_id', int),
('ra', float), ('dec', float),
('redshift', float), ('sn_truth_params', str, 500),
('is_sprinkled', int)])
sn_base_query = 'SELECT uniqueId, galaxy_id, '
sn_base_query += 'raJ2000, decJ2000, '
sn_base_query += 'redshift, sn_truth_params, is_sprinkled '
sn_base_query += 'FROM zpoint WHERE is_sn=1 '
filter_to_int = {'u':0, 'g':1, 'r':2, 'i':3, 'z':4, 'y':5}
n_floats = 0
with sqlite3.connect(out_file_name) as conn:
cursor = conn.cursor()
t_before_htmid = time.time()
# loop over trixels containing variables and transients, simulating
# the light curves of those objects
for htmid_dex, htmid in enumerate(object_htmid):
if htmid_dex>0:
htmid_duration = (time.time()-t_before_htmid)/3600.0
htmid_prediction = len(object_htmid)*htmid_duration/htmid_dex
print('%d htmid out of %d in %e hours; predict %e hours remaining' %
(htmid_dex, len(object_htmid), htmid_duration,htmid_prediction-htmid_duration))
mjd_arr = []
obs_arr = []
filter_arr = []
# Find only those pointings which overlap the current trixel
for obshistid in htmid_dict:
is_contained = False
for bounds in htmid_dict[obshistid]:
if htmid<=bounds[1] and htmid>=bounds[0]:
is_contained = True
break
if is_contained:
mjd_arr.append(mjd_dict[obshistid])
obs_arr.append(obshistid)
filter_arr.append(filter_to_int[filter_dict[obshistid]])
if len(mjd_arr) == 0:
continue
mjd_arr = np.array(mjd_arr)
obs_arr = np.array(obs_arr)
filter_arr = np.array(filter_arr)
sorted_dex = np.argsort(mjd_arr)
mjd_arr = mjd_arr[sorted_dex]
obs_arr = obs_arr[sorted_dex]
filter_arr = filter_arr[sorted_dex]
agn_query = agn_base_query + 'AND htmid=%d' % htmid
agn_iter = db.get_arbitrary_chunk_iterator(agn_query,
dtype=agn_dtype,
chunk_size=10000)
# put static data about the AGN (position, etc.) into the
# variables_and_transients table
for i_chunk, agn_results in enumerate(agn_iter):
values = ((int(agn_results['uniqueId'][i_obj]),
int(agn_results['galaxy_id'][i_obj]),
np.degrees(agn_results['ra'][i_obj]),
np.degrees(agn_results['dec'][i_obj]),
int(agn_results['is_sprinkled'][i_obj]),
1,0)
for i_obj in range(len(agn_results)))
cursor.executemany('''INSERT INTO variables_and_transients VALUES
(?,?,?,?,?,?,?)''', values)
agn_simulator = AgnSimulator(agn_results['redshift'])
quiescent_mag = np.zeros((len(agn_results), 6), dtype=float)
for i_obj, (sed_name, zz, mm) in enumerate(zip(agn_results['sed'],
agn_results['redshift'],
agn_results['magnorm'])):
spec = Sed()
spec.readSED_flambda(os.path.join(sed_dir, sed_name))
fnorm = getImsimFluxNorm(spec, mm)
spec.multiplyFluxNorm(fnorm)
spec.redshiftSED(zz, dimming=True)
mag_list = bp_dict.magListForSed(spec)
quiescent_mag[i_obj] = mag_list
# simulate AGN variability
dmag = agn_simulator.applyVariability(agn_results['varParamStr'],
expmjd=mjd_arr)
# loop over pointings that overlap the current trixel, writing
# out simulated photometry for each AGN observed in that pointing
for i_time, obshistid in enumerate(obs_arr):
# only include objects that were actually on a detector
are_on_chip = _actually_on_chip(np.degrees(agn_results['ra']),
np.degrees(agn_results['dec']),
obsmd_dict[obshistid])
valid_agn = np.where(are_on_chip)
if len(valid_agn[0])==0:
continue
values = ((int(agn_results['uniqueId'][i_obj]),
int(obs_arr[i_time]),
quiescent_mag[i_obj][filter_arr[i_time]]+
dmag[filter_arr[i_time]][i_obj][i_time])
for i_obj in valid_agn[0])
cursor.executemany('''INSERT INTO light_curves VALUES
(?,?,?)''', values)
conn.commit()
n_floats += len(dmag.flatten())
sn_query = sn_base_query + 'AND htmid=%d' % htmid
sn_iter = db.get_arbitrary_chunk_iterator(sn_query,
dtype=sn_dtype,
chunk_size=10000)
for sn_results in sn_iter:
t0_sne = time.time()
# write static information about SNe to the
# variables_and_transients table
values = ((int(sn_results['uniqueId'][i_obj]),
int(sn_results['galaxy_id'][i_obj]),
np.degrees(sn_results['ra'][i_obj]),
np.degrees(sn_results['dec'][i_obj]),
int(sn_results['is_sprinkled'][i_obj]),
0,1)
for i_obj in range(len(sn_results)))
cursor.executemany('''INSERT INTO variables_and_transients VALUES
(?,?,?,?,?,?,?)''', values)
conn.commit()
sn_mags = sn_simulator.calculate_sn_magnitudes(sn_results['sn_truth_params'],
mjd_arr, filter_arr)
print(' did %d sne in %e seconds' % (len(sn_results), time.time()-t0_sne))
# loop over pointings that overlap the current trixel, writing
# out simulated photometry for each SNe observed in that pointing
for i_time, obshistid in enumerate(obs_arr):
# only include objects that fell on a detector
are_on_chip = _actually_on_chip(np.degrees(sn_results['ra']),
np.degrees(sn_results['dec']),
obsmd_dict[obshistid])
valid_obj = np.where(np.logical_and(np.isfinite(sn_mags[:,i_time]),
are_on_chip))
if len(valid_obj[0]) == 0:
continue
values = ((int(sn_results['uniqueId'][i_obj]),
int(obs_arr[i_time]),
sn_mags[i_obj][i_time])
for i_obj in valid_obj[0])
cursor.executemany('''INSERT INTO light_curves VALUES (?,?,?)''', values)
conn.commit()
n_floats += len(valid_obj[0])
cursor.execute('CREATE INDEX unq_obs ON light_curves (uniqueId, obshistid)')
conn.commit()
print('n_floats %d' % n_floats)
print('in %e seconds' % (time.time()-t0_master))
boundType='circle',
boundLength=0.1)
data_iter = db.query_columns(col_names,
obs_metadata=obs,
chunk_size=100,
constraint="var_type=2")
for chunk in data_iter:
for star in chunk:
sed = Sed()
sed.readSED_flambda(
os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'],
defaultSpecMap[star['sedFilename']]))
fnorm = getImsimFluxNorm(sed, star['magNorm'])
sed.multiplyFluxNorm(fnorm)
mags = lsst_bp.magListForSed(sed)
du_nodust = np.abs(mags[0] - star['umag'])
dg_nodust = np.abs(mags[1] - star['gmag'])
dr_nodust = np.abs(mags[2] - star['rmag'])
di_nodust = np.abs(mags[3] - star['imag'])
dz_nodust = np.abs(mags[4] - star['zmag'])
dy_nodust = np.abs(mags[5] - star['ymag'])
a_x, b_x = sed.setupCCM_ab()
sed.addDust(a_x, b_x, R_v=3.1, ebv=star['ebv'])
mags = lsst_bp.magListForSed(sed)
du = np.abs(mags[0] - star['umag'])
dg = np.abs(mags[1] - star['gmag'])
delta_ast = rng.normal(0.0, ast_err_rad, n_stars)
(ra_smeared,
dec_smeared) = smear_ra_dec(ra, dec, delta_ast, rng)
delta_mag = rng.normal(0.0, phot_err, size=(n_stars, 6))
mags = np.zeros((n_stars, 6), dtype=float)
for ii in range(n_stars):
spec = photUtils.Sed()
file_name = os.path.join(sed_dir,
defaultSpecMap[sed_name[ii]])
spec.readSED_flambda(file_name)
if not np.array_equal(spec.wavelen, dust_wav):
a_x, b_x = spec.setupCCMab()
dust_wav = np.copy(spec.wavelen)
fnorm = photUtils.getImsimFluxNorm(spec, mag_norm[ii])
spec.multiplyFluxNorm(fnorm)
spec.addDust(a_x, b_x, ebv=ebv[ii], R_v=3.1)
mags[ii] = bp_dict.magListForSed(spec)
for ii in range(n_stars):
if mags[ii][2] > r_mag_limit:
continue
out_file.write('%d, %.10f, %.10f, ' %
(unq[ii], ra[ii], dec[ii]))
out_file.write('%.12f, %.12f, ' %
(ast_err_deg, ast_err_deg))
out_file.write('%.10f, %.10f, ' %
(ra_smeared[ii], dec_smeared[ii]))
for ibp in range(6):
out_file.write('%.5f, %.3f, ' %
