def dflux_for_kepler(chunk, filter_obs, mjd_obs, v_cache, dflux_out):
valid_obj = np.where(chunk['var_type']==1)
n_obj = len(valid_obj[0])
if n_obj==0:
return
params = {}
params['lc'] = chunk['lc_id'][valid_obj]
params['t0'] = chunk['t0'][valid_obj]
plc_model = ParametrizedLightCurveMixin()
dmag = plc_model.singleBandParametrizedLightCurve(np.array([True]*n_obj),
params, mjd_obs,
variability_cache=v_cache)
dummy_sed = Sed()
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
n_t = len(valid_obs[0])
if n_t==0:
continue
flux0 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj])
dmag_obs = dmag[:,valid_obs[0]].transpose()
assert dmag_obs.shape == (n_t, n_obj)
flux1 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj]
+ dmag_obs)
dflux_local = (flux1-flux0).transpose()
for i_local, i_global in enumerate(valid_obj[0]):
dflux_out[i_global][valid_obs] = dflux_local[i_local]
def dflux_for_rrly(chunk, filter_obs, mjd_obs, v_cache, dflux_out):
valid_obj = np.where(chunk['var_type'] == 3)
n_obj = len(valid_obj[0])
if n_obj == 0:
return
print('running RRLy')
params = {}
params['filename'] = np.array(
[v_cache['rrly_map'][ii] for ii in chunk['lc_id'][valid_obj]])
params['tStartMjd'] = chunk['t0'][valid_obj]
stellar_model = StellarVariabilityModels()
stellar_model.initializeVariability(doCache=True)
dmag = stellar_model.applyRRly(np.where(np.array([True] * n_obj)),
params,
mjd_obs,
variability_cache=v_cache).transpose(
0, 2, 1)
dummy_sed = Sed()
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
if len(valid_obs[0]) == 0:
continue
dmag_this_band = dmag[i_bp]
dmag_subset = dmag_this_band[valid_obs, :][0]
flux_0 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj])
flux_1 = dummy_sed.fluxFromMag(flux_0 + dmag_subset).transpose()
assert flux_1.shape == (n_obj, len(valid_obs[0]))
for i_local, i_global in enumerate(valid_obj[0]):
dflux = flux_1[i_local] - flux_0[i_local]
dflux_out[i_global][valid_obs] = dflux
def parse_kplr(chunk):
if not hasattr(parse_kplr, 'dmag_lookup'):
t_start = time.time()
fname = os.path.join(os.environ['SIMS_DATA_DIR'], 'catUtilsData',
'kplr_dmag_171204.txt')
if not os.path.isfile(fname):
raise RuntimeError('\n\n%s\nis not a file\n' % fname)
dtype = np.dtype([('lc_id', int), ('dmag', float)])
parse_kplr.dmag_lookup = np.genfromtxt(fname, dtype=dtype)
sorted_dex = np.argsort(parse_kplr.dmag_lookup['lc_id'])
parse_kplr.dmag_lookup = parse_kplr.dmag_lookup[sorted_dex]
parse_kplr.dflux_lookup = {}
with h5py.File('data/dflux_SNR5_lookup.h5','r') as snr_dflux_file:
for kk in snr_dflux_file.keys():
parse_kplr.dflux_lookup[kk] = snr_dflux_file[kk].value
#print('init %e' % (time.time()-t_start))
dummy_sed = Sed()
is_var = np.zeros(len(chunk), dtype=int)
dmag_dex = np.searchsorted(parse_kplr.dmag_lookup['lc_id'], chunk['lc_id'])
np.testing.assert_array_equal(chunk['lc_id'], parse_kplr.dmag_lookup['lc_id'][dmag_dex])
dmag_max = parse_kplr.dmag_lookup['dmag'][dmag_dex]
for bp in 'ugrizy':
dflux_threshold = np.interp(chunk[bp],
parse_kplr.dflux_lookup['mag_grid'],
parse_kplr.dflux_lookup['%s_dflux' % bp])
flux0 = dummy_sed.fluxFromMag(chunk[bp])
flux1 = dummy_sed.fluxFromMag(chunk[bp]+dmag_max)
dflux = np.abs(flux0-flux1)
local_is_var = (dflux>=dflux_threshold)
is_var[local_is_var] = 1
return is_var
def test_mags_vs_flux(self):
"""
Verify that the relationship between Sed.calcMag() and Sed.calcFlux()
is as expected
"""
wavelen = np.arange(100.0, 1500.0, 1.0)
flambda = np.exp(-0.5*np.power((wavelen-500.0)/100.0,2))
sb = (wavelen-100.0)/1400.0
ss = Sed(wavelen=wavelen, flambda=flambda)
bp = Bandpass(wavelen=wavelen, sb=sb)
mag = ss.calcMag(bp)
flux = ss.calcFlux(bp)
self.assertAlmostEqual(ss.magFromFlux(flux)/mag, 1.0, 10)
self.assertAlmostEqual(ss.fluxFromMag(mag)/flux, 1.0, 10)
def test_mags_vs_flux(self):
"""
Verify that the relationship between Sed.calcMag() and Sed.calcFlux()
is as expected
"""
wavelen = np.arange(100.0, 1500.0, 1.0)
flambda = np.exp(-0.5 * np.power((wavelen - 500.0) / 100.0, 2))
sb = (wavelen - 100.0) / 1400.0
ss = Sed(wavelen=wavelen, flambda=flambda)
bp = Bandpass(wavelen=wavelen, sb=sb)
mag = ss.calcMag(bp)
flux = ss.calcFlux(bp)
self.assertAlmostEqual(ss.magFromFlux(flux) / mag, 1.0, 10)
self.assertAlmostEqual(ss.fluxFromMag(mag) / flux, 1.0, 10)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs,
coadd_m5, m5_single, obs_md_list, proper_chip,
variability_cache, out_data, lock):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None]*n_t
dflux = np.zeros((n_obj,n_t), dtype=float)
dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_q = np.zeros((6,n_obj), dtype=float)
flux_coadd = np.zeros((6,n_obj), dtype=float)
mag_coadd = np.zeros((6,n_obj), dtype=float)
snr_coadd = np.zeros((6,n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1,
exptime=30.0)
t_start_snr = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0*coadd_visits[bp])
flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp])
dflux_sub = dflux[:,valid_obs[0]]
assert dflux_sub.shape == (n_obj, len(valid_obs[0]))
dflux_mean = np.mean(dflux_sub, axis=1)
assert dflux_mean.shape==(n_obj,)
flux_coadd[i_bp] = flux_q[i_bp]+dflux_mean
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp],
gamma) = SNR.calcSNR_m5(mag_coadd[i_bp],
lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp],
gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp],
m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
if len(valid_obs[0])==0:
continue
n_bp = len(valid_obs[0])
dflux_bp = dflux[:,valid_obs[0]]
flux0_arr = flux_q[i_bp]
flux_tot = flux0_arr[:,None] + dflux_bp
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot,
snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp]
noise_single = flux_tot/snr_single_val
noise = np.sqrt(noise_coadd[:,None]**2+noise_single**2)
dflux_thresh = 5.0*noise
dflux_bp = np.abs(dflux_bp)
detected = (dflux_bp>=dflux_thresh)
snr_arr[:,valid_obs[0]] = dflux_bp/noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj,valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'],
photometry_mask_1d, obs_md_list,
filter_obs, proper_chip)
duration = (time.time()-t_before_chip)/3600.0
unq_out = -1*np.ones(n_obj, dtype=int)
mjd_out = -1.0*np.ones(n_obj, dtype=float)
snr_out = -1.0*np.ones(n_obj, dtype=float)
var_type_out = -1*np.ones(n_obj, dtype=int)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:]
if detected.any():
unq_out[i_obj] = chunk['simobjid'][i_obj]
first_dex = np.where(detected)[0].min()
mjd_out[i_obj] = mjd_obs[first_dex]
snr_out[i_obj] = snr_arr[i_obj, first_dex]
var_type_out[i_obj] = chunk['var_type'][i_obj]
valid = np.where(unq_out>=0)
unq_out = unq_out[valid]
mjd_out = mjd_out[valid]
var_type_out = var_type_out[valid]
snr_out = snr_out[valid]
with lock:
existing_keys = list(out_data.keys())
key_val = 0
if len(existing_keys)>0:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None, None)
out_data[key_val] = (unq_out, mjd_out,
snr_out, var_type_out)
############ get true values of magnitudes from extragalactic catalog;
############ adjust magNorm to demand agreement
q_list = ['galaxy_id', 'ra', 'dec', 'redshift']
for bp in 'ugrizy':
q_list.append('mag_true_%s_lsst' % bp)
control_qties = cat.get_quantities(q_list, native_filters=[h_query])
for kk in control_qties:
control_qties[kk] = control_qties[kk][:args.lim]
np.testing.assert_array_equal(control_qties['galaxy_id'], disk_id)
dummy_spec = Sed()
for i_bp, bp in enumerate('ugrizy'):
cosmodc2_flux = dummy_spec.fluxFromMag(
control_qties['mag_true_%s_lsst' % bp])
flux_ratio = cosmodc2_flux / tot_lsst_fluxes[i_bp]
dmag = -2.5 * np.log10(flux_ratio)
disk_magnorm[i_bp, :] += dmag
bulge_magnorm[i_bp, :] += dmag
############# save everything in an hdf5 file
sed_dir = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], 'galaxySED')
list_of_seds = os.listdir(sed_dir)
list_of_seds.sort()
sed_names = np.empty(len(list_of_seds), dtype=(bytes, 100))
sed_idx = np.empty(len(list_of_seds), dtype=int)
name_to_int = {}
for ii, name in enumerate(list_of_seds):
full_name = os.path.join('galaxySED', name)
def test_selective_MLT_magnitudes(self):
"""
This test will verify that applyMLTflaring responds correctly
when mag_name_tuple is set to simulate only a subset of bandpasses
"""
rng =np.random.RandomState(87124)
n_stars = 1000
n_time = 5
quiescent_mags = {}
quiescent_mags['u'] = rng.random_sample(n_stars)*5.0+17.0
quiescent_mags['g'] = rng.random_sample(n_stars)*5.0+17.0
ebv = rng.random_sample(n_stars)*4.0
parallax = rng.random_sample(n_stars)*np.radians(0.001/3600.0)
available_lc = np.array(['lc_1', 'lc_2'])
params = {}
params['lc'] = rng.choice(available_lc, size=n_stars)
params['t0'] = rng.random_sample(n_stars)*1000.0
mjd_arr = rng.random_sample(n_time)*500.0*59580.0
mlt_obj = MLTflaringMixin()
mlt_obj.photParams = PhotometricParameters()
mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mlt_obj._mlt_lc_file = self.mlt_lc_name
mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g']
valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc']
valid_obj = rng.choice(np.arange(n_stars, dtype=int), size=50)
for ix in range(n_stars):
if ix not in valid_obj:
params['lc'][ix] = None
delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags)
n_time = len(mjd_arr)
self.assertEqual(delta_mag_vector.shape, (6, n_stars, n_time))
# test that delta_flux is correctly calculated
delta_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
do_mags=False)
self.assertEqual(delta_flux_vector.shape, delta_mag_vector.shape)
dummy_sed = Sed()
for i_band in range(6):
if i_band>=2:
# because we only implemented models of variability for u, g
np.testing.assert_array_equal(delta_flux_vector[i_band],
np.zeros(delta_flux_vector[i_band].shape, dtype=float))
continue
for i_obj in range(n_stars):
mag0 = quiescent_mags['ug'[i_band]][i_obj]
flux0 = dummy_sed.fluxFromMag(mag0)
delta_flux_control = dummy_sed.fluxFromMag(mag0 + delta_mag_vector[i_band][i_obj])-flux0
denom = np.abs(delta_flux_control)
denom = np.where(denom>0.0, denom, 1.0e-20)
err = np.abs(delta_flux_vector[i_band][i_obj]-delta_flux_control)/denom
self.assertLess(err.max(), 1.0e-6)
# see that we get the same results as before when we request a subset
# of magnitudes
test_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
mag_name_tuple=('g','i'))
non_zero = np.where(test_mag_vector[0]<0.0)
self.assertGreater(len(non_zero[0]), 0)
np.testing.assert_array_equal(test_mag_vector[0],
delta_mag_vector[1])
np.testing.assert_array_equal(test_mag_vector[1],
delta_mag_vector[3])
test_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
do_mags=False,
mag_name_tuple=('u','z','g'))
non_zero = np.where(test_flux_vector[2]>0.0)
self.assertGreater(len(non_zero[0]), 0)
np.testing.assert_array_equal(test_flux_vector[0],
delta_flux_vector[0])
np.testing.assert_array_equal(test_flux_vector[1],
delta_flux_vector[4])
np.testing.assert_array_equal(test_flux_vector[2],
delta_flux_vector[1])
def test_MLT_many_mjd_some_invalid(self):
"""
This test will verify that applyMLTflaring responds properly when given
an array/vector of MJD values in the case where some stars are not
marked as valid MLT stars.
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
rng = np.random.RandomState(16)
mjd_arr = rng.random_sample(17)*3653.3+59580.0
objid = []
parallax = []
ebv = []
quiescent_u = []
quiescent_g = []
delta_u = []
delta_g = []
varparams = []
for mjd in mjd_arr:
obs = ObservationMetaData(mjd=mjd)
cat = FlaringCatalog(db, obs_metadata=obs,
column_outputs=['parallax', 'ebv',
'quiescent_lsst_u',
'quiescent_lsst_g',
'varParamStr',
'delta_lsst_u',
'delta_lsst_g'])
cat._mlt_lc_file = self.mlt_lc_name
n_obj = 0
for line in cat.iter_catalog():
n_obj += 1
objid.append(line[0])
parallax.append(line[3])
ebv.append(line[4])
quiescent_u.append(line[5])
quiescent_g.append(line[6])
varparams.append(line[7])
delta_u.append(line[8])
delta_g.append(line[9])
objid = np.array(objid)
parallax = np.array(parallax)
ebv = np.array(ebv)
quiescent_u = np.array(quiescent_u)
quiescent_g = np.array(quiescent_g)
delta_u = np.array(delta_u)
delta_g = np.array(delta_g)
self.assertEqual(len(parallax), n_obj*len(mjd_arr))
np.testing.assert_array_equal(objid, np.array([0,1,2,3]*len(mjd_arr)))
quiescent_mags = {}
quiescent_mags['u'] = quiescent_u
quiescent_mags['g'] = quiescent_g
params = {}
params['lc'] = []
params['t0'] = []
for ix in range(n_obj):
local_dict = json.loads(varparams[ix])
params['lc'].append(local_dict['p']['lc'])
params['t0'].append(local_dict['p']['t0'])
params['lc'] = np.array(params['lc'])
params['t0'] = np.array(params['t0'])
mlt_obj = MLTflaringMixin()
mlt_obj.photParams = PhotometricParameters()
mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles()
mlt_obj._mlt_lc_file = cat._mlt_lc_file
mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g']
valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc']
for ix in range(n_obj):
if ix not in (1,2):
params['lc'][ix] = None
delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags)
n_time = len(mjd_arr)
self.assertEqual(delta_mag_vector.shape, (6, n_obj, n_time))
for i_time, mjd in enumerate(mjd_arr):
for i_obj in range(n_obj):
if i_obj not in (1,2):
for i_band in range(6):
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0)
continue
for i_band in range(6):
if i_band==0:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
delta_u[i_time*n_obj + i_obj])
elif i_band==1:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time],
delta_g[i_time*n_obj + i_obj])
else:
self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0)
# test that delta_flux is correctly calculated
delta_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr,
parallax=parallax,
ebv=ebv,
quiescent_mags=quiescent_mags,
do_mags=False)
self.assertEqual(delta_flux_vector.shape, delta_mag_vector.shape)
dummy_sed = Sed()
for i_band in range(6):
if i_band>=2:
# because we only implemented models of variability for u, g
np.testing.assert_array_equal(delta_flux_vector[i_band],
np.zeros(delta_flux_vector[i_band].shape, dtype=float))
continue
for i_obj in range(n_obj):
mag0 = quiescent_mags['ug'[i_band]][i_obj]
flux0 = dummy_sed.fluxFromMag(mag0)
delta_flux_control = dummy_sed.fluxFromMag(mag0 + delta_mag_vector[i_band][i_obj])-flux0
denom = np.abs(delta_flux_control)
denom = np.where(denom>0.0, denom, 1.0e-20)
err = np.abs(delta_flux_vector[i_band][i_obj]-delta_flux_control)/denom
self.assertLess(err.max(), 1.0e-10)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5,
obs_md_list, proper_chip, variability_cache,
out_data):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
# from the overview paper
# table 2; take m5 row and add Delta m5 row
# to get down to airmass 1.2
m5_single = {}
m5_single['u'] = 23.57
m5_single['g'] = 24.65
m5_single['r'] = 24.21
m5_single['i'] = 23.79
m5_single['z'] = 23.21
m5_single['y'] = 22.31
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None] * n_t
dflux = np.zeros((n_obj, n_t), dtype=float)
dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_q = np.zeros((6, n_obj), dtype=float)
flux_coadd = np.zeros((6, n_obj), dtype=float)
mag_coadd = np.zeros((6, n_obj), dtype=float)
snr_coadd = np.zeros((6, n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1, exptime=30.0)
t_start_snr = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0 *
coadd_visits[bp])
flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp])
dflux_sub = dflux[:, valid_obs[0]]
assert dflux_sub.shape == (n_obj, len(valid_obs[0]))
dflux_mean = np.mean(dflux_sub, axis=1)
assert dflux_mean.shape == (n_obj, )
flux_coadd[i_bp] = flux_q[i_bp] + dflux_mean
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp], m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
if len(valid_obs[0]) == 0:
continue
n_bp = len(valid_obs[0])
dflux_bp = dflux[:, valid_obs[0]]
flux0_arr = flux_q[i_bp]
flux_tot = flux0_arr[:, None] + dflux_bp
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot, snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp] / snr_coadd[i_bp]
noise_single = flux_tot / snr_single_val
noise = np.sqrt(noise_coadd[:, None]**2 + noise_single**2)
dflux_thresh = 5.0 * noise
dflux_bp = np.abs(dflux_bp)
detected = (dflux_bp >= dflux_thresh)
snr_arr[:, valid_obs[0]] = dflux_bp / noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj, valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'],
photometry_mask_1d, obs_md_list, filter_obs,
proper_chip)
duration = (time.time() - t_before_chip) / 3600.0
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj, :] & chip_mask[i_obj, :]
if detected.any():
unq = chunk['simobjid'][i_obj]
first_dex = np.where(detected)[0].min()
out_data[unq] = (mjd_obs[first_dex], snr_arr[i_obj, first_dex],
chunk['var_type'][i_obj])
if detected[0]:
ct_first += 1
else:
ct_at_all += 1
var_dict = json.loads(star['varParamStr'])
lc = var_dict['p']['lc']
lc_int_dex[i_star] = lc_to_int[lc]
for lc_id in np.unique(lc_int_dex):
valid_stars = np.where(lc_int_dex == lc_id)
if len(valid_stars[0]) == 0:
continue
local_flux_factor = flux_factor[valid_stars]
local_dmag_max = np.zeros(len(valid_stars[0]), dtype=float)
local_mag_min = 41.0 * np.ones(len(valid_stars[0]), dtype=float)
for mag_name in mag_name_list:
mag_0 = chunk['%smag' % mag_name][valid_stars]
local_dust_factor = dust_factor[mag_name][valid_stars]
flux_0 = dummy_sed.fluxFromMag(mag_0)
dflux = dflux_dict[lc_id][
mag_name] * local_dust_factor * local_flux_factor
total_flux = flux_0 + dflux
mag = dummy_sed.magFromFlux(total_flux)
dmag = 2.5 * np.log10(1.0 + dflux / flux_0)
local_dmag_max = np.where(dmag > local_dmag_max, dmag,
local_dmag_max)
local_mag_min = np.where(mag < local_mag_min, mag,
local_mag_min)
dmag_max[valid_stars] = local_dmag_max
mag_min[valid_stars] = local_mag_min
ct_total += len(chunk)
with open(out_name, 'a') as out_file:
params['t0'] = np.array([0.0] * n_steps)
q_mags = {}
for bp in 'ugrizy':
q_mags[bp] = quiet_mag * np.ones(n_steps, dtype=float)
for ebv_val in ebv_grid:
ebv = np.array([ebv_val] * n_steps, dtype=float)
dmag_base = mlt_model.applyMLTflaring(np.where(
np.array(n_steps * [True])),
params,
t_max,
parallax=parallax_grid,
ebv=ebv,
quiescent_mags=q_mags)
mag_base = dmag_base + quiet_mag
flux1 = dummy_sed.fluxFromMag(mag_base)
flux0 = dummy_sed.fluxFromMag(quiet_mag)
dflux = np.abs(flux1 - flux0)
for ii, bp in enumerate('ugrizy'):
dflux_name = '%s_ebv_%.2f_%s' % (lc_name, ebv_val, bp)
assert dflux_name not in out_cache
out_cache[dflux_name] = dflux[ii]
out_name = 'data/mlt_dflux_lookup.h5'
with h5py.File(out_name, 'w') as out_file:
for name in out_cache:
out_file.create_dataset(name, data=out_cache[name])
def test_avro_alert_generation_diff_dmag(self):
"""
Make sure everything works properly when the AlertDataGenerator
and the AvroAlertGenerator have different dmag thresholds
"""
dmag_cutoff_sqlite = 0.005
dmag_cutoff_avro = 0.2
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
star_db = StarAlertTestDBObj_avro(database=self.star_db_name,
driver='sqlite')
# assemble a dict of all of the alerts that need to be generated
obshistid_list = []
for obs in self.obs_list:
obshistid_list.append(obs.OpsimMetaData['obsHistID'])
obshistid_max = max(obshistid_list)
obshistid_bits = int(np.ceil(np.log(obshistid_max) / np.log(2.0)))
true_alert_dict = {}
obs_dict = {}
ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
cat.camera = obs_lsst_phosim.PhosimMapper().camera
for line in cat.iter_catalog():
if line[1] is None:
continue
dmag = line[2]
mag = line[3]
if (np.abs(dmag) > dmag_cutoff_avro and mag <=
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
alertId = (line[0] << obshistid_bits) + obshistid
self.assertNotIn(alertId, true_alert_dict)
true_alert_dict[alertId] = {}
true_alert_dict[alertId]['chipName'] = line[1]
true_alert_dict[alertId]['dmag'] = dmag
true_alert_dict[alertId]['mag'] = mag
true_alert_dict[alertId]['ra'] = np.degrees(line[4])
true_alert_dict[alertId]['decl'] = np.degrees(line[5])
true_alert_dict[alertId]['xPix'] = line[6]
true_alert_dict[alertId]['yPix'] = line[7]
elif np.abs(dmag) > dmag_cutoff_sqlite:
ignored_sqlite += 1
self.assertGreater(len(true_alert_dict), 10)
self.assertGreater(ignored_sqlite,
50) # just make sure that some sqlite
# alerts were ignored by the more
# stringent avro cut
log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir,
suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(
htmid,
star_db,
photometry_class=TestAlertsVarCat_avro,
output_prefix='alert_test',
output_dir=self.alert_data_output_dir,
dmag_cutoff=dmag_cutoff_sqlite,
log_file_name=log_file_name)
obshistid_to_htmid = {}
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid not in obshistid_to_htmid:
obshistid_to_htmid[obshistid] = []
obshistid_to_htmid[obshistid].append(htmid)
avro_gen = AvroAlertGenerator()
avro_gen.load_schema(
os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData',
'avroSchema'))
sql_prefix_list = ['alert_test']
out_prefix = 'test_avro'
log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
prefix='test_avro',
suffix='log.txt')
for obshistid in obshistid_list:
avro_gen.write_alerts(obshistid,
self.alert_data_output_dir,
sql_prefix_list,
obshistid_to_htmid[obshistid],
self.avro_out_dir,
out_prefix,
dmag_cutoff_avro,
lock=None,
log_file_name=log_file_name)
list_of_avro_files = os.listdir(self.avro_out_dir)
self.assertGreater(len(list_of_avro_files), 2)
alert_ct = 0
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
photParams = PhotometricParameters()
diasourceId_set = set()
for avro_file_name in list_of_avro_files:
if avro_file_name.endswith('log.txt'):
continue
full_name = os.path.join(self.avro_out_dir, avro_file_name)
with DataFileReader(open(full_name, 'rb'),
DatumReader()) as data_reader:
for alert in data_reader:
alert_ct += 1
obshistid = alert['alertId'] >> 20
obs = obs_dict[obshistid]
uniqueId = alert['diaObject']['diaObjectId']
true_alert_id = (uniqueId << obshistid_bits) + obshistid
self.assertIn(true_alert_id, true_alert_dict)
self.assertEqual(alert['l1dbId'], uniqueId)
true_alert = true_alert_dict[true_alert_id]
diaSource = alert['diaSource']
self.assertAlmostEqual(diaSource['ra'], true_alert['ra'],
10)
self.assertAlmostEqual(diaSource['decl'],
true_alert['decl'], 10)
self.assertAlmostEqual(diaSource['x'], true_alert['xPix'],
3)
self.assertAlmostEqual(diaSource['y'], true_alert['yPix'],
3)
self.assertAlmostEqual(diaSource['midPointTai'],
obs.mjd.TAI, 4)
true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
true_q_mag = true_alert['mag'] - true_alert['dmag']
true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
true_dflux = true_tot_flux - true_q_flux
self.assertAlmostEqual(diaSource['psFlux'] / true_dflux,
1.0, 6)
self.assertAlmostEqual(
diaSource['totFlux'] / true_tot_flux, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFlux'] / true_dflux,
1.0, 6)
true_tot_snr, gamma = calcSNR_m5(true_alert['mag'],
bp_dict[obs.bandpass],
obs.m5[obs.bandpass],
photParams)
true_q_snr, gamma = calcSNR_m5(
true_q_mag, bp_dict[obs.bandpass],
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
photParams)
true_tot_err = true_tot_flux / true_tot_snr
true_q_err = true_q_flux / true_q_snr
true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
self.assertAlmostEqual(
diaSource['snr'] / np.abs(true_dflux / true_diff_err),
1.0, 6)
self.assertAlmostEqual(
diaSource['totFluxErr'] / true_tot_err, 1.0, 6)
self.assertAlmostEqual(
diaSource['diffFluxErr'] / true_diff_err, 1.0, 6)
chipnum = int(true_alert['chipName'].replace(
'R', '').replace('S', '').replace(',', '').replace(
':', '').replace(' ', ''))
true_ccdid = (chipnum * 10**7) + obshistid
self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
self.assertEqual(uniqueId, diaSource['diaObjectId'])
self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
diasourceId_set.add(diaSource['diaSourceId'])
diaObject = alert['diaObject']
obj_dex = (uniqueId // 1024) - 1
self.assertAlmostEqual(
0.001 * diaObject['pmRa'] / self.pmra_truth[obj_dex],
1.0, 5)
self.assertAlmostEqual(
0.001 * diaObject['pmDecl'] /
self.pmdec_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(
0.001 * diaObject['parallax'] / self.px_truth[obj_dex],
1.0, 5)
(true_ra_base, true_dec_base) = applyProperMotion(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
self.assertEqual(alert_ct, len(true_alert_dict))
# in cm)
flux_factor = params.effarea/(4.0*np.pi*np.power(dd*3.08576e18, 2))
types_simulated = []
du = []
dg = []
dr = []
di = []
dz = []
dy = []
from mdwarf_utils import light_curve_from_class
from lsst.sims.photUtils import Sed
ss = Sed()
base_u = ss.fluxFromMag(data['umag'])
base_g = ss.fluxFromMag(data['gmag'])
base_r = ss.fluxFromMag(data['rmag'])
base_i = ss.fluxFromMag(data['imag'])
base_z = ss.fluxFromMag(data['zmag'])
base_y = ss.fluxFromMag(data['ymag'])
for i_star in range(len(data)):
thermo_class = type_dict[prob_vec[i_star]]
if thermo_class == 'late':
activity = 'active'
else:
activity = activity_level[i_star]
flare_type = '%s_%s' % (thermo_class, activity)
if flare_type in types_simulated:
def process_agn_chunk(chunk, filter_obs, mjd_obs, m5_obs,
coadd_m5, m5_single,
obs_md_list, proper_chip, out_data,
lock):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
agn_model = ExtraGalacticVariabilityModels()
dust_model = EBVbase()
with h5py.File('data/ebv_grid.h5', 'r') as in_file:
ebv_grid = in_file['ebv_grid'].value
extinction_grid = in_file['extinction_grid'].value
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None]*n_t
params = {}
params['agn_sfu'] = chunk['agn_sfu']
params['agn_sfg'] = chunk['agn_sfg']
params['agn_sfr'] = chunk['agn_sfr']
params['agn_sfi'] = chunk['agn_sfi']
params['agn_sfz'] = chunk['agn_sfz']
params['agn_sfy'] = chunk['agn_sfy']
params['agn_tau'] = chunk['agn_tau']
params['seed'] = chunk['id']+1
ebv = dust_model.calculateEbv(equatorialCoordinates=np.array([np.radians(chunk['ra']),
np.radians(chunk['dec'])]),
interp=True)
for i_bp, bp in enumerate('ugrizy'):
extinction_values = np.interp(ebv, ebv_grid, extinction_grid[i_bp])
chunk['%s_ab' % bp] += extinction_values
chunk['AGNLSST%s' % bp] += extinction_values
dmag = agn_model.applyAgn(np.where(np.array([True]*len(chunk))),
params, mjd_obs,
redshift=chunk['redshift'])
dmag_mean = np.mean(dmag, axis=2)
assert dmag_mean.shape == (6,n_obj)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_gal = np.zeros((6,n_obj), dtype=float)
flux_agn_q = np.zeros((6,n_obj), dtype=float)
flux_coadd = np.zeros((6,n_obj), dtype=float)
mag_coadd = np.zeros((6,n_obj), dtype=float)
snr_coadd = np.zeros((6,n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1,
exptime=30.0)
t_start_snr = time.time()
for i_bp, bp in enumerate('ugrizy'):
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0*coadd_visits[bp])
flux_gal[i_bp] = dummy_sed.fluxFromMag(chunk['%s_ab' % bp])
flux_agn_q[i_bp] = dummy_sed.fluxFromMag(chunk['AGNLSST%s' % bp] +
dmag_mean[i_bp,:])
flux_coadd[i_bp] = flux_gal[i_bp]+flux_agn_q[i_bp]
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp],
gamma) = SNR.calcSNR_m5(mag_coadd[i_bp],
lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp],
gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp],
m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
n_bp = len(valid_obs[0])
if n_bp == 0:
continue
mag0_arr = chunk['AGNLSST%s' % bp]
dmag_bp = dmag[i_bp][:,valid_obs[0]]
assert dmag_bp.shape == (n_obj, n_bp)
agn_flux_tot = dummy_sed.fluxFromMag(mag0_arr[:,None]+dmag_bp)
q_flux = flux_agn_q[i_bp]
agn_dflux = np.abs(agn_flux_tot-q_flux[:,None])
flux_tot = flux_gal[i_bp][:, None] + agn_flux_tot
assert flux_tot.shape == (n_obj, n_bp)
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot,
snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp]
noise_single = flux_tot/snr_single_val
noise = np.sqrt(noise_coadd[:,None]**2 + noise_single**2)
dflux_thresh = 5.0*noise
detected = (agn_dflux>=dflux_thresh)
assert detected.shape == (n_obj, n_bp)
snr_arr[:,valid_obs[0]] = agn_dflux/noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj, valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['dec'],
photometry_mask_1d, obs_md_list,
filter_obs, proper_chip)
duration = (time.time()-t_before_chip)/3600.0
unq_out = -1*np.ones(n_obj, dtype=int)
mjd_out = -1.0*np.ones(n_obj, dtype=float)
snr_out = -1.0*np.ones(n_obj, dtype=float)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:]
if detected.any():
unq_out[i_obj] = chunk['galtileid'][i_obj]
first_dex = np.where(detected)[0].min()
mjd_out[i_obj] = mjd_obs[first_dex]
snr_out[i_obj] = snr_arr[i_obj, first_dex]
valid = np.where(unq_out>=0)
unq_out = unq_out[valid]
mjd_out = mjd_out[valid]
snr_out = snr_out[valid]
with lock:
existing_keys = list(out_data.keys())
if len(existing_keys) == 0:
key_val = 0
else:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None)
out_data[key_val] = (unq_out, mjd_out, snr_out)
def applyMLTflaring(self, valid_dexes, params, expmjd,
parallax=None, ebv=None, quiescent_mags=None):
"""
parallax, ebv, and quiescent_mags are optional kwargs for use if you are
calling this method outside the context of an InstanceCatalog (presumably
with a numpy array of expmjd)
parallax is the parallax of your objects in radians
ebv is the E(B-V) value for your objects
quiescent_mags is a dict keyed on ('u', 'g', 'r', 'i', 'z', 'y')
with the quiescent magnitudes of the objects
"""
if parallax is None:
parallax = self.column_by_name('parallax')
if ebv is None:
ebv = self.column_by_name('ebv')
global _MLT_LC_NPZ
global _MLT_LC_NPZ_NAME
global _MLT_LC_TIME_CACHE
global _MLT_LC_FLUX_CACHE
# this needs to occur before loading the MLT light curve cache,
# just in case the user wants to override the light curve cache
# file by hand before generating the catalog
if len(params) == 0:
return numpy.array([[],[],[],[],[],[]])
if quiescent_mags is None:
quiescent_mags = {}
for mag_name in ('u', 'g', 'r', 'i', 'z', 'y'):
if ('lsst_%s' % mag_name in self._actually_calculated_columns or
'delta_lsst_%s' % mag_name in self._actually_calculated_columns):
quiescent_mags[mag_name] = self.column_by_name('quiescent_lsst_%s' % mag_name)
if not hasattr(self, 'photParams'):
raise RuntimeError("To apply MLT dwarf flaring, your "
"InstanceCatalog must have a member variable "
"photParams which is an instantiation of the "
"class PhotometricParameters, which can be "
"imported from lsst.sims.photUtils. "
"This is so that your InstanceCatalog has "
"knowledge of the effective area of the LSST "
"mirror.")
if _MLT_LC_NPZ is None or _MLT_LC_NPZ_NAME != self._mlt_lc_file or _MLT_LC_NPZ.fid is None:
if not os.path.exists(self._mlt_lc_file):
catutils_scripts = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts')
raise RuntimeError("The MLT flaring light curve file:\n"
+ "\n%s\n" % self._mlt_lc_file
+ "\ndoes not exist."
+"\n\n"
+ "Go into %s " % catutils_scripts
+ "and run get_mdwarf_flares.sh "
+ "to get the data")
_MLT_LC_NPZ = numpy.load(self._mlt_lc_file)
sims_clean_up.targets.append(_MLT_LC_NPZ)
_MLT_LC_NPZ_NAME = self._mlt_lc_file
_MLT_LC_TIME_CACHE = {}
_MLT_LC_FLUX_CACHE = {}
if not hasattr(self, '_mlt_dust_lookup'):
# Construct a look-up table to determine the factor
# by which to multiply the flares' flux to account for
# dust as a function of E(B-V). Recall that we are
# modeling all MLT flares as 9000K blackbodies.
if not hasattr(self, 'lsstBandpassDict'):
raise RuntimeError('You are asking for MLT dwarf flaring '
'magnitudes in a catalog that has not '
'defined lsstBandpassDict. The MLT '
'flaring magnitudes model does not know '
'how to apply dust extinction to the '
'flares without the member variable '
'lsstBandpassDict being defined.')
ebv_grid = numpy.arange(0.0, 7.01, 0.01)
bb_wavelen = numpy.arange(200.0, 1500.0, 0.1)
hc_over_k = 1.4387e7 # nm*K
temp = 9000.0 # black body temperature in Kelvin
exp_arg = hc_over_k/(temp*bb_wavelen)
exp_term = 1.0/(numpy.exp(exp_arg) - 1.0)
ln_exp_term = numpy.log(exp_term)
# Blackbody f_lambda function;
# discard normalizing factors; we only care about finding the
# ratio of fluxes between the case with dust extinction and
# the case without dust extinction
log_bb_flambda = -5.0*numpy.log(bb_wavelen) + ln_exp_term
bb_flambda = numpy.exp(log_bb_flambda)
bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda)
base_fluxes = self.lsstBandpassDict.fluxListForSed(bb_sed)
a_x, b_x = bb_sed.setupCCMab()
self._mlt_dust_lookup = {}
self._mlt_dust_lookup['ebv'] = ebv_grid
list_of_bp = self.lsstBandpassDict.keys()
for bp in list_of_bp:
self._mlt_dust_lookup[bp] = numpy.zeros(len(ebv_grid))
for iebv, ebv_val in enumerate(ebv_grid):
wv, fl = bb_sed.addCCMDust(a_x, b_x,
ebv=ebv_val,
wavelen=bb_wavelen,
flambda=bb_flambda)
dusty_bb = Sed(wavelen=wv, flambda=fl)
dusty_fluxes = self.lsstBandpassDict.fluxListForSed(dusty_bb)
for ibp, bp in enumerate(list_of_bp):
self._mlt_dust_lookup[bp][iebv] = dusty_fluxes[ibp]/base_fluxes[ibp]
# get the distance to each star in parsecs
_au_to_parsec = 1.0/206265.0
dd = _au_to_parsec/parallax
# get the area of the sphere through which the star's energy
# is radiating to get to us (in cm^2)
_cm_per_parsec = 3.08576e16
sphere_area = 4.0*numpy.pi*numpy.power(dd*_cm_per_parsec, 2)
flux_factor = self.photParams.effarea/sphere_area
if isinstance(expmjd, numbers.Number):
dMags = numpy.zeros((6, self.num_variable_obj(params)))
else:
dMags = numpy.zeros((6, self.num_variable_obj(params), len(expmjd)))
mag_name_tuple = ('u', 'g', 'r', 'i', 'z', 'y')
base_fluxes = {}
base_mags = {}
ss = Sed()
for mag_name in mag_name_tuple:
if ('lsst_%s' % mag_name in self._actually_calculated_columns or
'delta_lsst_%s' % mag_name in self._actually_calculated_columns):
mm = quiescent_mags[mag_name]
base_mags[mag_name] = mm
base_fluxes[mag_name] = ss.fluxFromMag(mm)
lc_name_arr = params['lc'].astype(str)
lc_names_unique = numpy.unique(lc_name_arr)
for lc_name in lc_names_unique:
if 'None' in lc_name:
continue
use_this_lc = numpy.where(numpy.char.find(lc_name_arr, lc_name)==0)
lc_name = lc_name.replace('.txt', '')
# 2017 May 1
# There isn't supposed to be a 'late_inactive' light curve.
# Unfortunately, I (Scott Daniel) assigned 'late_inactive'
# light curves to some of the stars on our database. Rather
# than fix the database table (which will take about a week of
# compute time), I am going to fix the problem here by mapping
# 'late_inactive' into 'late_active'.
if 'late' in lc_name:
lc_name = lc_name.replace('in', '')
if lc_name in _MLT_LC_TIME_CACHE:
raw_time_arr = _MLT_LC_TIME_CACHE[lc_name]
else:
raw_time_arr = _MLT_LC_NPZ['%s_time' % lc_name]
_MLT_LC_TIME_CACHE[lc_name] = raw_time_arr
time_arr = self._survey_start + raw_time_arr
dt = time_arr.max() - time_arr.min()
if isinstance(expmjd, numbers.Number):
t_interp = (expmjd + params['t0'][use_this_lc]).astype(float)
else:
n_obj = len(use_this_lc[0])
n_time = len(expmjd)
t_interp = numpy.ones(shape=(n_obj, n_time))*expmjd
t_interp += numpy.array([[tt]*n_time for tt in params['t0'][use_this_lc].astype(float)])
while t_interp.max() > time_arr.max():
bad_dexes = numpy.where(t_interp>time_arr.max())
t_interp[bad_dexes] -= dt
for i_mag, mag_name in enumerate(mag_name_tuple):
if ('lsst_%s' % mag_name in self._actually_calculated_columns or
'delta_lsst_%s' % mag_name in self._actually_calculated_columns):
flux_name = '%s_%s' % (lc_name, mag_name)
if flux_name in _MLT_LC_FLUX_CACHE:
flux_arr = _MLT_LC_FLUX_CACHE[flux_name]
else:
flux_arr = _MLT_LC_NPZ[flux_name]
_MLT_LC_FLUX_CACHE[flux_name] = flux_arr
dflux = numpy.interp(t_interp, time_arr, flux_arr)
if isinstance(expmjd, numbers.Number):
dflux *= flux_factor[use_this_lc]
else:
dflux *= numpy.array([flux_factor[use_this_lc]]*n_time).transpose()
dust_factor = numpy.interp(ebv[use_this_lc],
self._mlt_dust_lookup['ebv'],
self._mlt_dust_lookup[mag_name])
if not isinstance(expmjd, numbers.Number):
dust_factor = numpy.array([dust_factor]*n_time).transpose()
dflux *= dust_factor
if isinstance(expmjd, numbers.Number):
local_base_fluxes = base_fluxes[mag_name][use_this_lc]
local_base_mags = base_mags[mag_name][use_this_lc]
else:
local_base_fluxes = numpy.array([base_fluxes[mag_name][use_this_lc]]*n_time).transpose()
local_base_mags = numpy.array([base_mags[mag_name][use_this_lc]]*n_time).transpose()
dMags[i_mag][use_this_lc] = (ss.magFromFlux(local_base_fluxes + dflux)
- local_base_mags)
return dMags
def test_avro_alert_generation_diff_dmag(self):
"""
Make sure everything works properly when the AlertDataGenerator
and the AvroAlertGenerator have different dmag thresholds
"""
dmag_cutoff_sqlite = 0.005
dmag_cutoff_avro = 0.2
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite')
# assemble a dict of all of the alerts that need to be generated
obshistid_list = []
for obs in self.obs_list:
obshistid_list.append(obs.OpsimMetaData['obsHistID'])
obshistid_max = max(obshistid_list)
obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0)))
true_alert_dict = {}
obs_dict = {}
ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs)
cat.camera = lsst_camera()
for line in cat.iter_catalog():
if line[1] is None:
continue
dmag = line[2]
mag = line[3]
if (np.abs(dmag) > dmag_cutoff_avro and
mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]):
alertId = (line[0] << obshistid_bits) + obshistid
self.assertNotIn(alertId, true_alert_dict)
true_alert_dict[alertId] = {}
true_alert_dict[alertId]['chipName'] = line[1]
true_alert_dict[alertId]['dmag'] = dmag
true_alert_dict[alertId]['mag'] = mag
true_alert_dict[alertId]['ra'] = np.degrees(line[4])
true_alert_dict[alertId]['decl'] = np.degrees(line[5])
true_alert_dict[alertId]['xPix'] = line[6]
true_alert_dict[alertId]['yPix'] = line[7]
elif np.abs(dmag) > dmag_cutoff_sqlite:
ignored_sqlite += 1
self.assertGreater(len(true_alert_dict), 10)
self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite
# alerts were ignored by the more
# stringent avro cut
log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat_avro,
output_prefix='alert_test',
output_dir=self.alert_data_output_dir,
dmag_cutoff=dmag_cutoff_sqlite,
log_file_name=log_file_name)
obshistid_to_htmid = {}
for htmid in alert_gen.htmid_list:
for obs in alert_gen.obs_from_htmid(htmid):
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid not in obshistid_to_htmid:
obshistid_to_htmid[obshistid] = []
obshistid_to_htmid[obshistid].append(htmid)
avro_gen = AvroAlertGenerator()
avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema'))
sql_prefix_list = ['alert_test']
out_prefix = 'test_avro'
log_file_name = tempfile.mktemp(dir=self.avro_out_dir,
prefix='test_avro',
suffix='log.txt')
for obshistid in obshistid_list:
avro_gen.write_alerts(obshistid, self.alert_data_output_dir,
sql_prefix_list,
obshistid_to_htmid[obshistid],
self.avro_out_dir, out_prefix,
dmag_cutoff_avro, lock=None,
log_file_name=log_file_name)
list_of_avro_files = os.listdir(self.avro_out_dir)
self.assertGreater(len(list_of_avro_files), 2)
alert_ct = 0
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
photParams = PhotometricParameters()
diasourceId_set = set()
for avro_file_name in list_of_avro_files:
if avro_file_name.endswith('log.txt'):
continue
full_name = os.path.join(self.avro_out_dir, avro_file_name)
with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader:
for alert in data_reader:
alert_ct += 1
obshistid = alert['alertId'] >> 20
obs = obs_dict[obshistid]
uniqueId = alert['diaObject']['diaObjectId']
true_alert_id = (uniqueId << obshistid_bits) + obshistid
self.assertIn(true_alert_id, true_alert_dict)
self.assertEqual(alert['l1dbId'], uniqueId)
true_alert = true_alert_dict[true_alert_id]
diaSource = alert['diaSource']
self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10)
self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10)
self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3)
self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3)
self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4)
true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag'])
true_q_mag = true_alert['mag'] - true_alert['dmag']
true_q_flux = dummy_sed.fluxFromMag(true_q_mag)
true_dflux = true_tot_flux - true_q_flux
self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6)
self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6)
true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass],
obs.m5[obs.bandpass], photParams)
true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass],
self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]],
photParams)
true_tot_err = true_tot_flux/true_tot_snr
true_q_err = true_q_flux/true_q_snr
true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2)
self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err),
1.0, 6)
self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6)
self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6)
chipnum = int(true_alert['chipName'].replace('R', '').replace('S', '').
replace(',', '').replace(':', '').replace(' ', ''))
true_ccdid = (chipnum*10**7)+obshistid
self.assertEqual(true_ccdid, diaSource['ccdVisitId'])
self.assertEqual(uniqueId, diaSource['diaObjectId'])
self.assertNotIn(diaSource['diaSourceId'], diasourceId_set)
diasourceId_set.add(diaSource['diaSourceId'])
diaObject = alert['diaObject']
obj_dex = (uniqueId//1024) - 1
self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5)
self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5)
(true_ra_base,
true_dec_base) = applyProperMotion(self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=ModifiedJulianDate(TAI=diaObject['radecTai']))
self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7)
self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7)
self.assertEqual(alert_ct, len(true_alert_dict))
