def __init__(self, sed_name, max_mag=1000.):
"""
Read in the unnormalized SED.
"""
sed_dir = lsstUtils.getPackageDir('sims_sed_library')
self.sed_unnormed = photUtils.Sed()
self.sed_unnormed.readSED_flambda(os.path.join(sed_dir, sed_name))
self.max_mag = max_mag
def process_component(sed_names, ebv_arr, redshift, sed_dexes, av, rv, magnorm,
my_lock, output_dict, galaxy_id):
(dummy, hw_bp_dict) = sims_photUtils.BandpassDict.loadBandpassesFromFiles()
n_obj = len(sed_dexes)
fluxes = np.zeros((6, n_obj), dtype=float)
fluxes_noMW = np.zeros((6, n_obj), dtype=float)
t_start = time.time()
for sed_id in np.unique(sed_dexes):
valid = np.where(sed_dexes == sed_id)
sed_full_name = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'],
sed_names[sed_id])
spec_rest = sims_photUtils.Sed()
spec_rest.readSED_flambda(sed_full_name, cache_sed=False)
a_x_rest, b_x_rest = spec_rest.setupCCM_ab()
fnorm_at_21 = sims_photUtils.getImsimFluxNorm(spec_rest, 21.0)
for g_dex in valid[0]:
spec_rest_dust = sims_photUtils.Sed(wavelen=spec_rest.wavelen,
flambda=spec_rest.flambda)
spec_rest_dust.addDust(a_x_rest,
b_x_rest,
A_v=av[g_dex],
R_v=rv[g_dex])
spec_rest_dust.redshiftSED(redshift[g_dex], dimming=True)
flux_list_noMW = hw_bp_dict.fluxListForSed(spec_rest_dust)
a_x, b_x = spec_rest_dust.setupCCM_ab()
spec_rest_dust.addDust(a_x, b_x, R_v=3.1, ebv=ebv_arr[g_dex])
flux_list = hw_bp_dict.fluxListForSed(spec_rest_dust)
for i_bp, bp in enumerate('ugrizy'):
factor = fnorm_at_21 * np.power(
10.0, -0.4 * (magnorm[i_bp][g_dex] - 21.0))
fluxes[i_bp][g_dex] = factor * flux_list[i_bp]
fluxes_noMW[i_bp][g_dex] = factor * flux_list_noMW[i_bp]
with my_lock as context:
print('adding %d to fluxes' % len(fluxes))
output_dict['fluxes'].append(fluxes)
output_dict['fluxes_noMW'].append(fluxes_noMW)
output_dict['galaxy_id'].append(galaxy_id)
def _compute_SED(self):
self._sed_obj = sims_photUtils.Sed()
self._sed_obj.readSED_flambda(self.sed_file)
fnorm = sims_photUtils.getImsimFluxNorm(self._sed_obj, self.mag_norm)
self._sed_obj.multiplyFluxNorm(fnorm)
if self.iAv != 0:
self.shared_resources['ccm_model'].add_dust(
self._sed_obj, self.iAv, self.iRv, 'intrinsic')
if self.redshift != 0:
self._sed_obj.redshiftSED(self.redshift, dimming=True)
self._sed_obj.resampleSED(wavelen_match=self.bp_dict.wavelenMatch)
if self.gAv != 0:
self.shared_resources['ccm_model'].add_dust(
self._sed_obj, self.gAv, self.gRv, 'Galactic')
def __init__(self, chunk):
"""
chunk = (simobjid, hpid, sedFilename, magNorm, ebv, varParamStr,
parallax, ra, dec)
"""
# initialize some member variables expected by the
# stellar variability models
self.photParams = sims_photUtils.PhotometricParameters(nexp=1,
exptime=30)
self.lsstBandpassDict = sims_photUtils.BandpassDict.loadTotalBandpassesFromFiles(
)
self._actually_calculated_columns = []
for bp in 'ugrizy':
self._actually_calculated_columns.append('lsst_%s' % bp)
sed_dir = os.environ['SIMS_SED_LIBRARY_DIR']
self.quiescent_mags = {}
for bp in 'ugrizy':
self.quiescent_mags[bp] = np.zeros(len(chunk), dtype=float)
self.parallax = np.zeros(len(chunk), dtype=float)
self.ebv = np.zeros(len(chunk), dtype=float)
self.varParamStr = np.empty(len(chunk), dtype=(str, 300))
self.simobjid = np.empty(len(chunk), dtype=int)
ccm_wav = None
# find the quiescent magnitudes of the stars
for i_star, star in enumerate(chunk):
self.simobjid[i_star] = int(star[0])
full_file_name = os.path.join(sed_dir, defaultSpecMap[star[2]])
spec = sims_photUtils.Sed()
spec.readSED_flambda(full_file_name)
fnorm = sims_photUtils.getImsimFluxNorm(spec, float(star[3]))
spec.multiplyFluxNorm(fnorm)
if ccm_wav is None or not np.array_equal(spec.wavelen, ccm_wav):
ccm_wav = np.copy(spec.wavelen)
a_x, b_x = spec.setupCCM_ab()
ebv_val = float(star[4])
spec.addDust(a_x, b_x, ebv=ebv_val, R_v=3.1)
mag_list = self.lsstBandpassDict.magListForSed(spec)
for i_bp, bp in enumerate('ugrizy'):
self.quiescent_mags[bp][i_star] = mag_list[i_bp]
self.ebv[i_star] = ebv_val
self.varParamStr[i_star] = star[5]
self.parallax[i_star] = float(star[6])
self.parallax *= (np.pi / 648000000.0) # convert from mas to radians
def _create_sed(self):
"""
Function to create the lsst.sims.photUtils Sed object.
"""
self.sed = sims_photUtils.Sed()
self.sed.readSED_flambda(self.sed_file)
fnorm = sims_photUtils.getImsimFluxNorm(self.sed, self.mag_norm)
self.sed.multiplyFluxNorm(fnorm)
if self.iAv != 0:
self.add_dust(self.iAv, self.iRv, 'intrinsic')
if self.redshift != 0:
self.sed.redshiftSED(self.redshift, dimming=True)
self.sed.resampleSED(wavelen_match=self.bp_dict.wavelenMatch)
if self.gAv != 0:
self.add_dust(self.gAv, self.gRv, 'Galactic')
def __init__(self, sed_name, max_mag=1000.):
"""
Set up the LSST bandpasses and un-normalized SED.
"""
self.bps = dict()
throughput_dir = lsstUtils.getPackageDir('throughputs')
for band in 'ugrizy':
self.bps[band] = photUtils.Bandpass()
self.bps[band].readThroughput(
os.path.join(throughput_dir, 'baseline',
'total_%s.dat' % band))
self.control_bandpass = photUtils.Bandpass()
self.control_bandpass.imsimBandpass()
sed_dir = lsstUtils.getPackageDir('sims_sed_library')
self.sed_unnormed = photUtils.Sed()
self.sed_unnormed.readSED_flambda(os.path.join(sed_dir, sed_name))
self.max_mag = max_mag
fluxes_noMW[component] = np.zeros(n_test_gals, dtype=float)
fluxes[component] = np.zeros(n_test_gals, dtype=float)
for component in ['disk', 'bulge']:
sed_arr = sed_fit_file['%s_sed' % component][()][subset]
av_arr = sed_fit_file['%s_av' % component][()][subset]
rv_arr = sed_fit_file['%s_rv' % component][()][subset]
mn_arr = sed_fit_file['%s_magnorm' %
component][()][i_bp, :][subset]
z_arr = cosmoDC2_data['redshift'][crossmatch_dex]
for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate(
zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)):
# read in the SED file from the library
sed_file_name = os.path.join(sed_lib_dir, sed_names[s_dex])
sed = sims_photUtils.Sed()
sed.readSED_flambda(sed_file_name)
# find and apply normalizing flux
fnorm = sims_photUtils.getImsimFluxNorm(sed, mn)
sed.multiplyFluxNorm(fnorm)
# add internal dust
if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w):
ccm_w = np.copy(sed.wavelen)
a_x, b_x = sed.setupCCM_ab()
sed.addDust(a_x, b_x, A_v=av, R_v=rv)
# apply redshift
sed.redshiftSED(zz, dimming=True)
def do_photometry(chunk, obs_lock, obs_metadata_dict, star_lock,
star_data_dict, job_lock, job_dict, out_dir):
"""
-Take a chunk from the stellar database
-Calculate the light curves for all of the stars in the chunk
-Write the light curves to disk
-Save the metadata and static data to dicts to be output later
Parameters
----------
chunk -- a set of stars returned by sqlite3.fetchmany()
the columns in chunk are:
simobjid, hpid, sedFilename, magNorm, ebv, varParamStr,
parallax, ra, decl
Note: chunk must be from one hpid (healpix pixel ID)
obs_lock -- the lock corresponding to the observation metadata
for this healpixel
obs_metadata_dict -- the dict where observation metadata is stored
star_lock -- the lock corresponding to the stellar metadata for this
healpixel
star_data_dict -- the dict where stellar metadata (i.e. static data) is
stored
job_lock -- the lock used to prevent too many processes from entering
light curve generation at once
job_dict -- the dict keeping track of how many jobs are doing light curve
generation
out_dir -- directory where light curve files are to be stored
Returns
-------
None
"""
dummy_sed = sims_photUtils.Sed()
# find the lookup file associating healpixel with obsHistID;
# this should probably actually be passed into the method
data_dir = '/astro/store/pogo4/danielsf/desc_dc2_truth'
assert os.path.isdir(data_dir)
hpid_lookup_name = os.path.join(data_dir, 'hpid_to_obsHistID_lookup.h5')
assert os.path.isfile(hpid_lookup_name)
metadata_dict = {}
metadata_keys = ['obsHistID', 'ra', 'dec', 'rotTelPos', 'mjd', 'filter']
hpid = int(chunk[0][1])
with h5py.File(hpid_lookup_name, 'r') as in_file:
valid_obsid = in_file['%d' % hpid][()]
for k in metadata_keys:
metadata_dict[k] = in_file[k][()]
valid_obsid = np.sort(valid_obsid)
valid_dex = np.searchsorted(metadata_dict['obsHistID'], valid_obsid)
np.testing.assert_array_equal(metadata_dict['obsHistID'][valid_dex],
valid_obsid)
for k in metadata_dict.keys():
metadata_dict[k] = metadata_dict[k][valid_dex]
# generate delta_magnitude light curves
has_dmag = False
while not has_dmag:
# make sure no more than 5 processes are doing this at once
with job_lock:
if job_dict['running_dmag'] >= 5:
continue
else:
job_dict['running_dmag'] += 1
#print('running dmag %d' % job_dict['running_dmag'])
t_start = time.time()
var_gen = VariabilityGenerator(chunk)
dmag_raw = var_gen.applyVariability(var_gen.varParamStr,
expmjd=metadata_dict['mjd'])
dmag_raw = dmag_raw.transpose([1, 2, 0])
# transpose so the columns are (star, mjd, filter)
assert dmag_raw.shape == (len(chunk), len(metadata_dict['mjd']), 6)
dmag = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=float)
for i_mjd in range(len(metadata_dict['mjd'])):
dmag[:, i_mjd] = dmag_raw[:, i_mjd, metadata_dict['filter'][i_mjd]]
del dmag_raw
has_dmag = True
with job_lock:
job_dict['running_dmag'] -= 1
#print('running dmag %d' % job_dict['running_dmag'])
quiescent_fluxes = np.zeros((len(chunk), 6), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
quiescent_fluxes[:, i_bp] = dummy_sed.fluxFromMag(
var_gen.column_by_name('quiescent_%s' % bp))
t_dmag = time.time() - t_start
star_ra = np.array([c[7] for c in chunk])
star_dec = np.array([c[8] for c in chunk])
# Now figure out which obsHistID are observing which stars
obs_mask = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=bool)
fov_radius = 2.1 # in degrees
t_start = time.time()
for i_obs in range(len(metadata_dict['mjd'])):
ra = np.degrees(metadata_dict['ra'][i_obs])
dec = np.degrees(metadata_dict['dec'][i_obs])
## back-of-the-envelope implementation
## (does not use focal plane model with chip gaps, etc.)
#ang_dist = angularSeparation(ra, dec, star_ra, star_dec)
#valid = ang_dist<fov_radius
rotTel = np.degrees(metadata_dict['rotTelPos'][i_obs])
obs_md = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
mjd=metadata_dict['mjd'][i_obs])
rotSky = getRotSkyPos(ra, dec, obs_md, rotTel)
obs_md.rotSkyPos = rotSky
chip_name = chipNameFromRaDecLSST(star_ra,
star_dec,
obs_metadata=obs_md).astype(str)
valid = np.char.find(chip_name, 'None') < 0
obs_mask[:, i_obs] = valid
# verify that any stars with empty light curves are, in fact
# outside of DC2
region_of_interest = DC2_bounds()
xyz_offenders = []
for i_star in range(len(chunk)):
if obs_mask[i_star].sum() == 0:
xyz_offenders.append(
xyz_from_ra_dec(star_ra[i_star], star_dec[i_star]))
if len(xyz_offenders) > 0:
xyz_offenders = np.array(xyz_offenders)
v0 = region_of_interest.hs_list[0].contains_many_pts(xyz_offenders)
v1 = region_of_interest.hs_list[1].contains_many_pts(xyz_offenders)
v2 = region_of_interest.hs_list[2].contains_many_pts(xyz_offenders)
v3 = region_of_interest.hs_list[3].contains_many_pts(xyz_offenders)
if (v0 & v1 & v2 & v3).sum() > 0:
msg = "\n\nsome stars in healpixel %d unobserved\n\n" % hpid
raise RuntimeError(msg)
t_flux = time.time()
dflux_arr = []
for i_star in range(len(chunk)):
star_filter = metadata_dict['filter'][obs_mask[i_star]]
q_flux = quiescent_fluxes[i_star][star_filter]
assert len(q_flux) == obs_mask[i_star].sum()
q_mag = dummy_sed.magFromFlux(q_flux)
tot_mag = q_mag + dmag[i_star, obs_mask[i_star]]
tot_flux = dummy_sed.fluxFromMag(tot_mag)
dflux = tot_flux - q_flux
dflux_arr.append(dflux)
assert len(dflux) == obs_mask[i_star].sum()
# store metadata in the output dicts
with obs_lock:
obs_metadata_dict['mjd'].append(metadata_dict['mjd'])
obs_metadata_dict['obsHistID'].append(metadata_dict['obsHistID'])
obs_metadata_dict['filter'].append(metadata_dict['filter'])
with star_lock:
local_simobjid = var_gen.column_by_name('simobjid')
out_file_name = create_out_name(out_dir, hpid)
# write the light curves to disk now; otherwise, the memory
# footprint of the job becomes too large
with h5py.File(out_file_name, 'a') as out_file:
for i_star in range(len(local_simobjid)):
if len(dflux_arr[i_star]) == 0:
continue
simobjid = local_simobjid[i_star]
out_file.create_dataset('%d_flux' % simobjid,
data=dflux_arr[i_star])
out_file.create_dataset(
'%d_obsHistID' % simobjid,
data=metadata_dict['obsHistID'][obs_mask[i_star]])
star_data_dict['simobjid'].append(local_simobjid)
star_data_dict['ra'].append(star_ra)
star_data_dict['dec'].append(star_dec)
for i_bp, bp in enumerate('ugrizy'):
star_data_dict['quiescent_%s' % bp].append(quiescent_fluxes[:,
i_bp])
def _process_chunk(db_lock, log_lock, sema, sed_fit_name, cosmoDC2_data,
first_gal, self_dict, bad_gals):
"""
Do all chunk-specific work: compute table contents for a
collection of galaxies and write to db
Parameters
----------
db_lock Used to avoid conflicts writing to sqlite output
log_lock Used to avoid conflicts writing to per-healpixel log
sema A semaphore. Release when done
sed_fit_name File where sed fits for this healpixel are
cosmoDC2_data Values from cosmoDC2 for this healpixel, keyed by
column name
first_gal index of first galaxy in our chunk (in sed fit list)
self_dict Random useful values stored in GalaxyTruthWriter
bad_gals List of galaxy ids, monotone increasing, to be
skipped
"""
dry = self_dict['dry']
chunk_size = self_dict['chunk_size']
dbfile = self_dict['dbfile']
logfile = self_dict['logfile']
if dry:
_logit(
log_lock, logfile,
'_process_chunk invoke for first_gal {}, chunk size {}'.format(
first_gal, chunk_size))
if sema is None:
return
sema.release()
#exit(0)
return
lsst_bp_dict = self_dict['lsst_bp_dict']
galaxy_ids = []
ra = []
dec = []
redshift = []
ebv_vals = None
ebv_vals_init = False # does this belong somewhere else?
ccm_w = None
total_gals = self_dict['total_gals']
chunk_start = first_gal
chunk_end = min(first_gal + chunk_size, total_gals)
with h5py.File(sed_fit_name, 'r') as sed_fit_file:
sed_names = sed_fit_file['sed_names'][()]
sed_names = [s.decode() for s in sed_names] # becse stored as bytes
gals_this_chunk = chunk_end - chunk_start
subset = slice(chunk_start, chunk_end)
galaxy_ids = sed_fit_file['galaxy_id'][()][subset]
to_log = 'Start with galaxy #{}, id={}\n# galaxies for _process_chunk: {}\n'.format(
first_gal, galaxy_ids[0], len(galaxy_ids))
_logit(log_lock, logfile, to_log)
# get the cross-match between the sed fit and cosmoDC2
cosmo_len = len(cosmoDC2_data['galaxy_id'])
crossmatch_dex = np.searchsorted(cosmoDC2_data['galaxy_id'],
galaxy_ids)
np.testing.assert_array_equal(
galaxy_ids, cosmoDC2_data['galaxy_id'][crossmatch_dex])
ra = sed_fit_file['ra'][()][subset]
dec = sed_fit_file['dec'][()][subset]
np.testing.assert_array_equal(ra, cosmoDC2_data['ra'][crossmatch_dex])
np.testing.assert_array_equal(dec,
cosmoDC2_data['dec'][crossmatch_dex])
good_ixes = _good_indices(galaxy_ids.tolist(), bad_gals[0])
if (len(good_ixes) == 0):
if sema is not None:
sema.release()
return
else:
_logit(
log_lock, logfile,
'Found {} good indices for chunk starting with {}\n'.format(
len(good_ixes), chunk_start))
flux_by_band_MW = {}
flux_by_band_noMW = {}
# Calculate E(B-V) for dust extinction in Milky Way along relevant
# lines of sight
band_print = "Processing band {}, first gal {}, time {}\n"
if not ebv_vals_init:
equatorial_coords = np.array([np.radians(ra), np.radians(dec)])
ebv_model = EBVbase()
ebv_vals = ebv_model.calculateEbv(
equatorialCoordinates=equatorial_coords, interp=True)
ebv_vals_init = True
for i_bp, bp in enumerate('ugrizy'):
if (i_bp == 0 or i_bp == 5):
_logit(log_lock, logfile,
band_print.format(bp, first_gal, dt.now()))
fluxes_noMW = {}
fluxes = {}
for component in ['disk', 'bulge']:
fluxes_noMW[component] = np.zeros(gals_this_chunk, dtype=float)
fluxes[component] = np.zeros(gals_this_chunk, dtype=float)
for component in ['disk', 'bulge']:
#print(" Processing component ", component)
sed_arr = sed_fit_file['%s_sed' % component][()][subset]
av_arr = sed_fit_file['%s_av' % component][()][subset]
rv_arr = sed_fit_file['%s_rv' % component][()][subset]
mn_arr = sed_fit_file['%s_magnorm' %
component][()][i_bp, :][subset]
z_arr = cosmoDC2_data['redshift'][crossmatch_dex]
gii = 0
done = False
for i_gal, (s_dex, mn, av, rv, zz, ebv) in enumerate(
zip(sed_arr, mn_arr, av_arr, rv_arr, z_arr, ebv_vals)):
if done: break
while good_ixes[gii] < i_gal:
gii += 1
if gii == len(good_ixes): # ran out of good ones
done = True
break
if done: break
if good_ixes[gii] > i_gal: # skipped over it; it's bad
continue
# Leave space for it in the arrays, but values
# for all the fluxes will be left at 0
# read in the SED file from the library
sed_file_name = os.path.join(self_dict['sed_lib_dir'],
sed_names[s_dex])
sed = sims_photUtils.Sed()
sed.readSED_flambda(sed_file_name)
# find and apply normalizing flux
fnorm = sims_photUtils.getImsimFluxNorm(sed, mn)
sed.multiplyFluxNorm(fnorm)
# add internal dust
if ccm_w is None or not np.array_equal(sed.wavelen, ccm_w):
ccm_w = np.copy(sed.wavelen)
a_x, b_x = sed.setupCCM_ab()
sed.addDust(a_x, b_x, A_v=av, R_v=rv)
# apply redshift
sed.redshiftSED(zz, dimming=True)
# flux, in Janskys, without Milky Way dust extinction
f_noMW = sed.calcFlux(lsst_bp_dict[bp])
# apply Milky Way dust
# (cannot reuse a_x, b_x because wavelength grid changed
# when we called redshiftSED)
a_x_mw, b_x_mw = sed.setupCCM_ab()
sed.addDust(a_x_mw, b_x_mw, R_v=3.1, ebv=ebv)
f_MW = sed.calcFlux(lsst_bp_dict[bp])
fluxes_noMW[component][i_gal] = f_noMW
fluxes[component][i_gal] = f_MW
if (component == 'disk') and (bp == 'r'):
redshift = z_arr
# Sum components and convert to nanojansky
total_fluxes = (fluxes_noMW['disk'] + fluxes_noMW['bulge']) * 10**9
total_fluxes_MW = (fluxes['disk'] + fluxes['bulge']) * 10**9
dummy_sed = sims_photUtils.Sed()
# add magnification due to weak lensing
kappa = cosmoDC2_data['convergence'][crossmatch_dex]
gamma_sq = (cosmoDC2_data['shear_1'][crossmatch_dex]**2 +
cosmoDC2_data['shear_2'][crossmatch_dex]**2)
magnification = 1.0 / ((1.0 - kappa)**2 - gamma_sq)
magnified_fluxes = magnification * total_fluxes
magnified_fluxes_MW = magnification * total_fluxes_MW
flux_by_band_noMW[bp] = magnified_fluxes
flux_by_band_MW[bp] = magnified_fluxes_MW
# Open connection to sqlite db and write
#print('Time before db write is {}, first gal={}'.format(dt.now(), first_gal))
#sys.stdout.flush()
if not db_lock.acquire(timeout=120.0):
_logit(log_lock, logfile, "Failed to acquire db lock, first gal=",
first_gal)
if sema is None:
return
sema.release()
exit(1)
try:
_write_sqlite(dbfile, galaxy_ids, ra, dec, redshift, flux_by_band_MW,
flux_by_band_noMW, good_ixes)
db_lock.release()
if sema is not None:
sema.release()
_logit(
log_lock, logfile,
'Time after db write: {}, first_gal={}\n'.format(
dt.now(), first_gal))
exit(0)
except Exception as ex:
db_lock.release()
if sema is not None:
sema.release()
raise (ex)
dec = star_r[2].astype(float)
sed_name = star_r[3].astype(str)
mag_norm = star_r[4].astype(float)
eq_coord = np.array([np.radians(ra), np.radians(dec)])
ebv = ebv_gen.calculateEbv(equatorialCoordinates=eq_coord,
interp=True)
delta_ast = rng.normal(0.0, ast_err_rad, n_stars)
(ra_smeared,
dec_smeared) = smear_ra_dec(ra, dec, delta_ast, rng)
delta_mag = rng.normal(0.0, phot_err, size=(n_stars, 6))
mags = np.zeros((n_stars, 6), dtype=float)
for ii in range(n_stars):
spec = photUtils.Sed()
file_name = os.path.join(sed_dir,
defaultSpecMap[sed_name[ii]])
spec.readSED_flambda(file_name)
if not np.array_equal(spec.wavelen, dust_wav):
a_x, b_x = spec.setupCCMab()
dust_wav = np.copy(spec.wavelen)
fnorm = photUtils.getImsimFluxNorm(spec, mag_norm[ii])
spec.multiplyFluxNorm(fnorm)
spec.addDust(a_x, b_x, ebv=ebv[ii], R_v=3.1)
mags[ii] = bp_dict.magListForSed(spec)
for ii in range(n_stars):
if mags[ii][2] > r_mag_limit:
continue
out_file.write('%d, %.10f, %.10f, ' %
def validate_sne(cat_dir, obsid, fov_deg=2.1,
scatter_file=None, vanishing_file=None):
"""
Parameters
----------
cat_dir is the parent dir of $obsid
obsid is the obsHistID of the pointing (an int)
fov_deg is the radius of the field of view in degrees
"""
project_dir = os.path.join('/global/projecta/projectdirs',
'lsst/groups/SSim/DC2/cosmoDC2_v1.1.4')
sne_db_name = os.path.join(project_dir, 'sne_cosmoDC2_v1.1.4_MS_DDF.db')
if not os.path.isfile(sne_db_name):
raise RuntimeError("\n%s\nis not a file" % sne_db_name)
instcat_dir = os.path.join(cat_dir, '%.8d' % obsid)
if not os.path.isdir(instcat_dir):
raise RuntimeError("\n%s\nis not a dir" % instcat_dir)
sne_name = os.path.join(instcat_dir, 'sne_cat_%d.txt.gz' % obsid)
if not os.path.isfile(sne_name):
raise RuntimeError("\n%s\nis not a file" % sne_name)
phosim_name = os.path.join(instcat_dir, 'phosim_cat_%d.txt' % obsid)
if not os.path.isfile(phosim_name):
raise RuntimeError("\n%s\nis not a file" % phosim_name)
sed_dir = os.path.join(instcat_dir, "Dynamic")
if not os.path.isdir(sed_dir):
raise RuntimeError("\n%s\nis not a dir" % sed_dir)
opsim_db = os.path.join("/global/projecta/projectdirs",
"lsst/groups/SSim/DC2",
"minion_1016_desc_dithered_v4_sfd.db")
if not os.path.isfile(opsim_db):
raise RuntimeError("\n%s\nis not a file" % opsim_db)
band_from_int = 'ugrizy'
bandpass = None
with open(phosim_name, 'r') as in_file:
for line in in_file:
params = line.strip().split()
if params[0] == 'filter':
bandpass = band_from_int[int(params[1])]
break
if bandpass is None:
raise RuntimeError("Failed to read in bandpass")
bp_dict = photUtils.BandpassDict.loadTotalBandpassesFromFiles()
with sqlite3.connect(opsim_db) as conn:
c = conn.cursor()
r = c.execute("SELECT descDitheredRA, descDitheredDec, expMJD FROM Summary "
"WHERE obsHistID==%d" % obsid).fetchall()
pointing_ra = np.degrees(r[0][0])
pointing_dec = np.degrees(r[0][1])
expmjd = float(r[0][2])
with sqlite3.connect(sne_db_name) as conn:
c = conn.cursor()
query = "SELECT snid_in, snra_in, sndec_in, "
query += "c_in, mB, t0_in, x0_in, x1_in, z_in "
query += "FROM sne_params WHERE ("
where_clause = None
half_space = htm.halfSpaceFromRaDec(pointing_ra, pointing_dec, fov_deg)
bounds = half_space.findAllTrixels(6)
for bound in bounds:
if where_clause is not None:
where_clause += " OR ("
else:
where_clause = "("
if bound[0] == bound[1]:
where_clause += "htmid_level_6==%d" % bound[0]
else:
where_clause += "htmid_level_6>=%d AND htmid_level_6<=%d" % (bound[0],bound[1])
where_clause += ")"
query += where_clause+")"
sn_params = c.execute(query).fetchall()
sn_ra = np.array([sn[1] for sn in sn_params])
sn_dec = np.array([sn[2] for sn in sn_params])
sn_d = angularSeparation(pointing_ra, pointing_dec,
sn_ra, sn_dec)
valid = np.where(sn_d<=fov_deg)
sn_ra_dec = {}
sn_param_dict = {}
for i_sn in valid[0]:
sn = sn_params[i_sn]
sn_param_dict[sn[0]] = sn[3:]
sn_ra_dec[sn[0]] = sn[1:3]
sne_in_instcat = set()
d_mag_max = -1.0
with gzip.open(sne_name, 'rb') as sne_cat_file:
for sne_line in sne_cat_file:
instcat_params = sne_line.strip().split(b' ')
sne_id = instcat_params[1].decode('utf-8')
if sne_id not in sn_param_dict:
try:
sne_id_int = int(sne_id)
assert sne_id_int<1.5e10
except (ValueError, AssertionError):
raise RuntimeError("\n%s\nnot in SNe db" % sne_id)
sne_in_instcat.add(sne_id)
control_params = sn_param_dict[sne_id]
sed_name = os.path.join(instcat_dir, instcat_params[5].decode('utf-8'))
if not os.path.isfile(sed_name):
raise RuntimeError("\n%s\nis not a file" % sed_name)
sed = photUtils.Sed()
sed.readSED_flambda(sed_name, cache_sed=False)
fnorm = photUtils.getImsimFluxNorm(sed, float(instcat_params[4]))
sed.multiplyFluxNorm(fnorm)
sed.redshiftSED(float(instcat_params[6]), dimming=True)
a_x, b_x = sed.setupCCM_ab()
A_v = float(instcat_params[15])
R_v = float(instcat_params[16])
EBV = A_v/R_v
sed.addDust(a_x, b_x, A_v=A_v, R_v=R_v)
sn_object = SNObject()
sn_object.set_MWebv(EBV)
sn_object.set(c=control_params[0], x1=control_params[4],
x0=control_params[3], t0=control_params[2],
z=control_params[5])
sn_mag = sn_object.catsimBandMag(bp_dict[bandpass], expmjd)
instcat_flux = sed.calcFlux(bp_dict[bandpass])
instcat_mag = sed.magFromFlux(instcat_flux)
d_mag = (sn_mag-instcat_mag)
if not np.isfinite(d_mag):
if np.isfinite(sn_mag) or np.isfinite(instcat_mag):
msg = '%s\ngave magnitudes %e %e (diff %e)' % (sne_id,
sn_mag,
instcat_mag,
d_mag)
print(msg)
if scatter_file is not None:
scatter_file.write("%e %e %e\n" % (sn_mag, instcat_mag, d_mag))
d_mag = np.abs(d_mag)
if d_mag>d_mag_max:
d_mag_max = d_mag
#print('dmag %e -- %e (%e)' %
# (d_mag, instcat_mag,
# control_params[5]-float(instcat_params[6])))
msg = '\n%s\n' % sne_name
ct_missing = 0
for sn_id in sn_param_dict:
if sn_id in sne_in_instcat:
continue
control_params = sn_param_dict[sn_id]
sn_object = SNObject()
sn_object.set_MWebv(0.0)
cc = control_params[0]
x1 = control_params[4]
x0 = control_params[3]
t0 = control_params[2]
zz = control_params[5]
sn_object.set(c=cc, x1=x1, x0=x0, t0=t0, z=zz)
sn_mag = sn_object.catsimBandMag(bp_dict[bandpass], expmjd)
if vanishing_file is not None and np.isfinite(sn_mag):
vanishing_file.write('%d %s %s %e ' % (obsid, bandpass, sn_id, sn_mag))
vanishing_file.write('%e %e %e %.4f %e %.4f %e\n' %
(cc,x0,x1,t0,zz,expmjd,expmjd-t0))
def validate_chunk(data_in,
in_dir, healpix,
read_lock, write_lock, output_dict):
galaxy_id = data_in['galaxy_id']
redshift = data_in['redshift']
mag_in = {}
for bp in 'ugrizy':
mag_in[bp] = data_in['mag_true_%s_lsst' % bp]
eps = 1.0e-20
disk_to_bulge = {}
for bp in 'ugrizy':
disk_name = 'LSST_filters/diskLuminositiesStellar:'
disk_name += 'LSST_%s:observed:dustAtlas' % bp
bulge_name = 'LSST_filters/spheroidLuminositiesStellar:'
bulge_name += 'LSST_%s:observed:dustAtlas' % bp
denom = np.where(data_in[bulge_name]>0.0, data_in[bulge_name], eps)
disk_to_bulge[bp] = data_in[disk_name]/denom
assert np.isfinite(disk_to_bulge[bp]).sum()==len(disk_to_bulge[bp])
sed_dir = os.environ['SIMS_SED_LIBRARY_DIR']
tot_bp_dict = photUtils.BandpassDict.loadTotalBandpassesFromFiles()
fit_file = os.path.join(in_dir, 'sed_fit_%d.h5' % healpix)
assert os.path.isfile(fit_file)
with read_lock:
with h5py.File(fit_file, 'r') as in_file:
sed_names = in_file['sed_names'][()]
galaxy_id_fit = in_file['galaxy_id'][()]
valid = np.in1d(galaxy_id_fit, galaxy_id)
galaxy_id_fit = galaxy_id_fit[valid]
np.testing.assert_array_equal(galaxy_id_fit, galaxy_id)
disk_magnorm = in_file['disk_magnorm'][()][:,valid]
disk_av = in_file['disk_av'][()][valid]
disk_rv = in_file['disk_rv'][()][valid]
disk_sed = in_file['disk_sed'][()][valid]
bulge_magnorm = in_file['bulge_magnorm'][()][:,valid]
bulge_av = in_file['bulge_av'][()][valid]
bulge_rv = in_file['bulge_rv'][()][valid]
bulge_sed = in_file['bulge_sed'][()][valid]
sed_names = [name.decode() for name in sed_names]
local_worst = {}
local_worst_ratio = {}
ccm_w = None
dummy_sed = photUtils.Sed()
for i_obj in range(len(disk_av)):
for i_bp, bp in enumerate('ugrizy'):
d_sed = photUtils.Sed()
fname = os.path.join(sed_dir, sed_names[disk_sed[i_obj]])
d_sed.readSED_flambda(fname)
fnorm = photUtils.getImsimFluxNorm(d_sed,
disk_magnorm[i_bp][i_obj])
d_sed.multiplyFluxNorm(fnorm)
if ccm_w is None or not np.array_equal(d_sed.wavelen, ccm_w):
ccm_w = np.copy(d_sed.wavelen)
ax, bx = d_sed.setupCCM_ab()
d_sed.addDust(ax, bx, R_v=disk_rv[i_obj], A_v=disk_av[i_obj])
d_sed.redshiftSED(redshift[i_obj], dimming=True)
d_flux = d_sed.calcFlux(tot_bp_dict[bp])
b_sed = photUtils.Sed()
fname = os.path.join(sed_dir, sed_names[bulge_sed[i_obj]])
b_sed.readSED_flambda(fname)
fnorm = photUtils.getImsimFluxNorm(b_sed,
bulge_magnorm[i_bp][i_obj])
b_sed.multiplyFluxNorm(fnorm)
if ccm_w is None or not np.array_equal(b_sed.wavelen, ccm_w):
ccm_w = np.copy(b_sed.wavelen)
ax, bx = b_sed.setupCCM_ab()
b_sed.addDust(ax, bx, R_v=bulge_rv[i_obj], A_v=bulge_av[i_obj])
b_sed.redshiftSED(redshift[i_obj], dimming=True)
b_flux = b_sed.calcFlux(tot_bp_dict[bp])
flux_ratio = d_flux/max(b_flux, eps)
if flux_ratio<1.0e3 or disk_to_bulge[bp][i_obj]<1.0e3:
delta_ratio = np.abs(flux_ratio-disk_to_bulge[bp][i_obj])
if disk_to_bulge[bp][i_obj]>eps:
delta_ratio /= disk_to_bulge[bp][i_obj]
if bp not in local_worst_ratio or delta_ratio>local_worst_ratio[bp]:
local_worst_ratio[bp] = delta_ratio
tot_flux = b_flux + d_flux
true_flux = dummy_sed.fluxFromMag(mag_in[bp][i_obj])
delta_flux = np.abs(1.0-tot_flux/true_flux)
if bp not in local_worst or delta_flux>local_worst[bp]:
local_worst[bp] = delta_flux
with write_lock:
for bp in 'ugrizy':
if bp not in output_dict or local_worst[bp]>output_dict[bp]:
output_dict[bp] = local_worst[bp]
ratio_name = '%s_ratio' % bp
if (ratio_name not in output_dict
or local_worst_ratio[bp]>output_dict[ratio_name]):
output_dict[ratio_name] = local_worst_ratio[bp]