def dflux_for_kepler(chunk, filter_obs, mjd_obs, v_cache, dflux_out): valid_obj = np.where(chunk['var_type']==1) n_obj = len(valid_obj[0]) if n_obj==0: return params = {} params['lc'] = chunk['lc_id'][valid_obj] params['t0'] = chunk['t0'][valid_obj] plc_model = ParametrizedLightCurveMixin() dmag = plc_model.singleBandParametrizedLightCurve(np.array([True]*n_obj), params, mjd_obs, variability_cache=v_cache) dummy_sed = Sed() for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs==i_bp) n_t = len(valid_obs[0]) if n_t==0: continue flux0 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj]) dmag_obs = dmag[:,valid_obs[0]].transpose() assert dmag_obs.shape == (n_t, n_obj) flux1 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj] + dmag_obs) dflux_local = (flux1-flux0).transpose() for i_local, i_global in enumerate(valid_obj[0]): dflux_out[i_global][valid_obs] = dflux_local[i_local]
def dflux_for_rrly(chunk, filter_obs, mjd_obs, v_cache, dflux_out): valid_obj = np.where(chunk['var_type'] == 3) n_obj = len(valid_obj[0]) if n_obj == 0: return print('running RRLy') params = {} params['filename'] = np.array( [v_cache['rrly_map'][ii] for ii in chunk['lc_id'][valid_obj]]) params['tStartMjd'] = chunk['t0'][valid_obj] stellar_model = StellarVariabilityModels() stellar_model.initializeVariability(doCache=True) dmag = stellar_model.applyRRly(np.where(np.array([True] * n_obj)), params, mjd_obs, variability_cache=v_cache).transpose( 0, 2, 1) dummy_sed = Sed() for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) if len(valid_obs[0]) == 0: continue dmag_this_band = dmag[i_bp] dmag_subset = dmag_this_band[valid_obs, :][0] flux_0 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj]) flux_1 = dummy_sed.fluxFromMag(flux_0 + dmag_subset).transpose() assert flux_1.shape == (n_obj, len(valid_obs[0])) for i_local, i_global in enumerate(valid_obj[0]): dflux = flux_1[i_local] - flux_0[i_local] dflux_out[i_global][valid_obs] = dflux
def parse_kplr(chunk): if not hasattr(parse_kplr, 'dmag_lookup'): t_start = time.time() fname = os.path.join(os.environ['SIMS_DATA_DIR'], 'catUtilsData', 'kplr_dmag_171204.txt') if not os.path.isfile(fname): raise RuntimeError('\n\n%s\nis not a file\n' % fname) dtype = np.dtype([('lc_id', int), ('dmag', float)]) parse_kplr.dmag_lookup = np.genfromtxt(fname, dtype=dtype) sorted_dex = np.argsort(parse_kplr.dmag_lookup['lc_id']) parse_kplr.dmag_lookup = parse_kplr.dmag_lookup[sorted_dex] parse_kplr.dflux_lookup = {} with h5py.File('data/dflux_SNR5_lookup.h5','r') as snr_dflux_file: for kk in snr_dflux_file.keys(): parse_kplr.dflux_lookup[kk] = snr_dflux_file[kk].value #print('init %e' % (time.time()-t_start)) dummy_sed = Sed() is_var = np.zeros(len(chunk), dtype=int) dmag_dex = np.searchsorted(parse_kplr.dmag_lookup['lc_id'], chunk['lc_id']) np.testing.assert_array_equal(chunk['lc_id'], parse_kplr.dmag_lookup['lc_id'][dmag_dex]) dmag_max = parse_kplr.dmag_lookup['dmag'][dmag_dex] for bp in 'ugrizy': dflux_threshold = np.interp(chunk[bp], parse_kplr.dflux_lookup['mag_grid'], parse_kplr.dflux_lookup['%s_dflux' % bp]) flux0 = dummy_sed.fluxFromMag(chunk[bp]) flux1 = dummy_sed.fluxFromMag(chunk[bp]+dmag_max) dflux = np.abs(flux0-flux1) local_is_var = (dflux>=dflux_threshold) is_var[local_is_var] = 1 return is_var
def test_mags_vs_flux(self): """ Verify that the relationship between Sed.calcMag() and Sed.calcFlux() is as expected """ wavelen = np.arange(100.0, 1500.0, 1.0) flambda = np.exp(-0.5*np.power((wavelen-500.0)/100.0,2)) sb = (wavelen-100.0)/1400.0 ss = Sed(wavelen=wavelen, flambda=flambda) bp = Bandpass(wavelen=wavelen, sb=sb) mag = ss.calcMag(bp) flux = ss.calcFlux(bp) self.assertAlmostEqual(ss.magFromFlux(flux)/mag, 1.0, 10) self.assertAlmostEqual(ss.fluxFromMag(mag)/flux, 1.0, 10)
def test_mags_vs_flux(self): """ Verify that the relationship between Sed.calcMag() and Sed.calcFlux() is as expected """ wavelen = np.arange(100.0, 1500.0, 1.0) flambda = np.exp(-0.5 * np.power((wavelen - 500.0) / 100.0, 2)) sb = (wavelen - 100.0) / 1400.0 ss = Sed(wavelen=wavelen, flambda=flambda) bp = Bandpass(wavelen=wavelen, sb=sb) mag = ss.calcMag(bp) flux = ss.calcFlux(bp) self.assertAlmostEqual(ss.magFromFlux(flux) / mag, 1.0, 10) self.assertAlmostEqual(ss.fluxFromMag(mag) / flux, 1.0, 10)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, m5_single, obs_md_list, proper_chip, variability_cache, out_data, lock): t_start_chunk = time.time() #print('processing %d' % len(chunk)) ct_first = 0 ct_at_all = 0 ct_tot = 0 n_t = len(filter_obs) n_obj = len(chunk) coadd_visits = {} coadd_visits['u'] = 6 coadd_visits['g'] = 8 coadd_visits['r'] = 18 coadd_visits['i'] = 18 coadd_visits['z'] = 16 coadd_visits['y'] = 16 gamma_coadd = {} for bp in 'ugrizy': gamma_coadd[bp] = None gamma_single = {} for bp in 'ugrizy': gamma_single[bp] = [None]*n_t dflux = np.zeros((n_obj,n_t), dtype=float) dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux) dummy_sed = Sed() lsst_bp = BandpassDict.loadTotalBandpassesFromFiles() flux_q = np.zeros((6,n_obj), dtype=float) flux_coadd = np.zeros((6,n_obj), dtype=float) mag_coadd = np.zeros((6,n_obj), dtype=float) snr_coadd = np.zeros((6,n_obj), dtype=float) snr_single = {} snr_single_mag_grid = np.arange(14.0, 30.0, 0.05) phot_params_single = PhotometricParameters(nexp=1, exptime=30.0) t_start_snr = time.time() photometry_mask = np.zeros((n_obj, n_t), dtype=bool) photometry_mask_1d = np.zeros(n_obj, dtype=bool) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) phot_params_coadd = PhotometricParameters(nexp=1, exptime=30.0*coadd_visits[bp]) flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp]) dflux_sub = dflux[:,valid_obs[0]] assert dflux_sub.shape == (n_obj, len(valid_obs[0])) dflux_mean = np.mean(dflux_sub, axis=1) assert dflux_mean.shape==(n_obj,) flux_coadd[i_bp] = flux_q[i_bp]+dflux_mean mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp]) (snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp], coadd_m5[bp], phot_params_coadd) (snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid, lsst_bp[bp], m5_single[bp], phot_params_single) #print('got all snr in %e' % (time.time()-t_start_snr)) t_start_obj = time.time() snr_arr = np.zeros((n_obj, n_t), dtype=float) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs==i_bp) if len(valid_obs[0])==0: continue n_bp = len(valid_obs[0]) dflux_bp = dflux[:,valid_obs[0]] flux0_arr = flux_q[i_bp] flux_tot = flux0_arr[:,None] + dflux_bp mag_tot = dummy_sed.magFromFlux(flux_tot) snr_single_val = np.interp(mag_tot, snr_single_mag_grid, snr_single[bp]) noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp] noise_single = flux_tot/snr_single_val noise = np.sqrt(noise_coadd[:,None]**2+noise_single**2) dflux_thresh = 5.0*noise dflux_bp = np.abs(dflux_bp) detected = (dflux_bp>=dflux_thresh) snr_arr[:,valid_obs[0]] = dflux_bp/noise for i_obj in range(n_obj): if detected[i_obj].any(): photometry_mask_1d[i_obj] = True photometry_mask[i_obj,valid_obs[0]] = detected[i_obj] t_before_chip = time.time() chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'], photometry_mask_1d, obs_md_list, filter_obs, proper_chip) duration = (time.time()-t_before_chip)/3600.0 unq_out = -1*np.ones(n_obj, dtype=int) mjd_out = -1.0*np.ones(n_obj, dtype=float) snr_out = -1.0*np.ones(n_obj, dtype=float) var_type_out = -1*np.ones(n_obj, dtype=int) for i_obj in range(n_obj): if photometry_mask_1d[i_obj]: detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:] if detected.any(): unq_out[i_obj] = chunk['simobjid'][i_obj] first_dex = np.where(detected)[0].min() mjd_out[i_obj] = mjd_obs[first_dex] snr_out[i_obj] = snr_arr[i_obj, first_dex] var_type_out[i_obj] = chunk['var_type'][i_obj] valid = np.where(unq_out>=0) unq_out = unq_out[valid] mjd_out = mjd_out[valid] var_type_out = var_type_out[valid] snr_out = snr_out[valid] with lock: existing_keys = list(out_data.keys()) key_val = 0 if len(existing_keys)>0: key_val = max(existing_keys) while key_val in existing_keys: key_val += 1 out_data[key_val] = (None, None, None, None) out_data[key_val] = (unq_out, mjd_out, snr_out, var_type_out)
############ get true values of magnitudes from extragalactic catalog; ############ adjust magNorm to demand agreement q_list = ['galaxy_id', 'ra', 'dec', 'redshift'] for bp in 'ugrizy': q_list.append('mag_true_%s_lsst' % bp) control_qties = cat.get_quantities(q_list, native_filters=[h_query]) for kk in control_qties: control_qties[kk] = control_qties[kk][:args.lim] np.testing.assert_array_equal(control_qties['galaxy_id'], disk_id) dummy_spec = Sed() for i_bp, bp in enumerate('ugrizy'): cosmodc2_flux = dummy_spec.fluxFromMag( control_qties['mag_true_%s_lsst' % bp]) flux_ratio = cosmodc2_flux / tot_lsst_fluxes[i_bp] dmag = -2.5 * np.log10(flux_ratio) disk_magnorm[i_bp, :] += dmag bulge_magnorm[i_bp, :] += dmag ############# save everything in an hdf5 file sed_dir = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], 'galaxySED') list_of_seds = os.listdir(sed_dir) list_of_seds.sort() sed_names = np.empty(len(list_of_seds), dtype=(bytes, 100)) sed_idx = np.empty(len(list_of_seds), dtype=int) name_to_int = {} for ii, name in enumerate(list_of_seds): full_name = os.path.join('galaxySED', name)
def test_selective_MLT_magnitudes(self): """ This test will verify that applyMLTflaring responds correctly when mag_name_tuple is set to simulate only a subset of bandpasses """ rng =np.random.RandomState(87124) n_stars = 1000 n_time = 5 quiescent_mags = {} quiescent_mags['u'] = rng.random_sample(n_stars)*5.0+17.0 quiescent_mags['g'] = rng.random_sample(n_stars)*5.0+17.0 ebv = rng.random_sample(n_stars)*4.0 parallax = rng.random_sample(n_stars)*np.radians(0.001/3600.0) available_lc = np.array(['lc_1', 'lc_2']) params = {} params['lc'] = rng.choice(available_lc, size=n_stars) params['t0'] = rng.random_sample(n_stars)*1000.0 mjd_arr = rng.random_sample(n_time)*500.0*59580.0 mlt_obj = MLTflaringMixin() mlt_obj.photParams = PhotometricParameters() mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles() mlt_obj._mlt_lc_file = self.mlt_lc_name mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g'] valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc'] valid_obj = rng.choice(np.arange(n_stars, dtype=int), size=50) for ix in range(n_stars): if ix not in valid_obj: params['lc'][ix] = None delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr, parallax=parallax, ebv=ebv, quiescent_mags=quiescent_mags) n_time = len(mjd_arr) self.assertEqual(delta_mag_vector.shape, (6, n_stars, n_time)) # test that delta_flux is correctly calculated delta_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr, parallax=parallax, ebv=ebv, quiescent_mags=quiescent_mags, do_mags=False) self.assertEqual(delta_flux_vector.shape, delta_mag_vector.shape) dummy_sed = Sed() for i_band in range(6): if i_band>=2: # because we only implemented models of variability for u, g np.testing.assert_array_equal(delta_flux_vector[i_band], np.zeros(delta_flux_vector[i_band].shape, dtype=float)) continue for i_obj in range(n_stars): mag0 = quiescent_mags['ug'[i_band]][i_obj] flux0 = dummy_sed.fluxFromMag(mag0) delta_flux_control = dummy_sed.fluxFromMag(mag0 + delta_mag_vector[i_band][i_obj])-flux0 denom = np.abs(delta_flux_control) denom = np.where(denom>0.0, denom, 1.0e-20) err = np.abs(delta_flux_vector[i_band][i_obj]-delta_flux_control)/denom self.assertLess(err.max(), 1.0e-6) # see that we get the same results as before when we request a subset # of magnitudes test_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr, parallax=parallax, ebv=ebv, quiescent_mags=quiescent_mags, mag_name_tuple=('g','i')) non_zero = np.where(test_mag_vector[0]<0.0) self.assertGreater(len(non_zero[0]), 0) np.testing.assert_array_equal(test_mag_vector[0], delta_mag_vector[1]) np.testing.assert_array_equal(test_mag_vector[1], delta_mag_vector[3]) test_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr, parallax=parallax, ebv=ebv, quiescent_mags=quiescent_mags, do_mags=False, mag_name_tuple=('u','z','g')) non_zero = np.where(test_flux_vector[2]>0.0) self.assertGreater(len(non_zero[0]), 0) np.testing.assert_array_equal(test_flux_vector[0], delta_flux_vector[0]) np.testing.assert_array_equal(test_flux_vector[1], delta_flux_vector[4]) np.testing.assert_array_equal(test_flux_vector[2], delta_flux_vector[1])
def test_MLT_many_mjd_some_invalid(self): """ This test will verify that applyMLTflaring responds properly when given an array/vector of MJD values in the case where some stars are not marked as valid MLT stars. """ db = MLT_test_DB(database=self.db_name, driver='sqlite') rng = np.random.RandomState(16) mjd_arr = rng.random_sample(17)*3653.3+59580.0 objid = [] parallax = [] ebv = [] quiescent_u = [] quiescent_g = [] delta_u = [] delta_g = [] varparams = [] for mjd in mjd_arr: obs = ObservationMetaData(mjd=mjd) cat = FlaringCatalog(db, obs_metadata=obs, column_outputs=['parallax', 'ebv', 'quiescent_lsst_u', 'quiescent_lsst_g', 'varParamStr', 'delta_lsst_u', 'delta_lsst_g']) cat._mlt_lc_file = self.mlt_lc_name n_obj = 0 for line in cat.iter_catalog(): n_obj += 1 objid.append(line[0]) parallax.append(line[3]) ebv.append(line[4]) quiescent_u.append(line[5]) quiescent_g.append(line[6]) varparams.append(line[7]) delta_u.append(line[8]) delta_g.append(line[9]) objid = np.array(objid) parallax = np.array(parallax) ebv = np.array(ebv) quiescent_u = np.array(quiescent_u) quiescent_g = np.array(quiescent_g) delta_u = np.array(delta_u) delta_g = np.array(delta_g) self.assertEqual(len(parallax), n_obj*len(mjd_arr)) np.testing.assert_array_equal(objid, np.array([0,1,2,3]*len(mjd_arr))) quiescent_mags = {} quiescent_mags['u'] = quiescent_u quiescent_mags['g'] = quiescent_g params = {} params['lc'] = [] params['t0'] = [] for ix in range(n_obj): local_dict = json.loads(varparams[ix]) params['lc'].append(local_dict['p']['lc']) params['t0'].append(local_dict['p']['t0']) params['lc'] = np.array(params['lc']) params['t0'] = np.array(params['t0']) mlt_obj = MLTflaringMixin() mlt_obj.photParams = PhotometricParameters() mlt_obj.lsstBandpassDict = BandpassDict.loadTotalBandpassesFromFiles() mlt_obj._mlt_lc_file = cat._mlt_lc_file mlt_obj._actually_calculated_columns = ['delta_lsst_u', 'delta_lsst_g'] valid_dex = [] # applyMLT does not actually use valid_dex; it looks for non-None params['lc'] for ix in range(n_obj): if ix not in (1,2): params['lc'][ix] = None delta_mag_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr, parallax=parallax, ebv=ebv, quiescent_mags=quiescent_mags) n_time = len(mjd_arr) self.assertEqual(delta_mag_vector.shape, (6, n_obj, n_time)) for i_time, mjd in enumerate(mjd_arr): for i_obj in range(n_obj): if i_obj not in (1,2): for i_band in range(6): self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0) continue for i_band in range(6): if i_band==0: self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], delta_u[i_time*n_obj + i_obj]) elif i_band==1: self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], delta_g[i_time*n_obj + i_obj]) else: self.assertEqual(delta_mag_vector[i_band][i_obj][i_time], 0.0) # test that delta_flux is correctly calculated delta_flux_vector = mlt_obj.applyMLTflaring(valid_dex, params, mjd_arr, parallax=parallax, ebv=ebv, quiescent_mags=quiescent_mags, do_mags=False) self.assertEqual(delta_flux_vector.shape, delta_mag_vector.shape) dummy_sed = Sed() for i_band in range(6): if i_band>=2: # because we only implemented models of variability for u, g np.testing.assert_array_equal(delta_flux_vector[i_band], np.zeros(delta_flux_vector[i_band].shape, dtype=float)) continue for i_obj in range(n_obj): mag0 = quiescent_mags['ug'[i_band]][i_obj] flux0 = dummy_sed.fluxFromMag(mag0) delta_flux_control = dummy_sed.fluxFromMag(mag0 + delta_mag_vector[i_band][i_obj])-flux0 denom = np.abs(delta_flux_control) denom = np.where(denom>0.0, denom, 1.0e-20) err = np.abs(delta_flux_vector[i_band][i_obj]-delta_flux_control)/denom self.assertLess(err.max(), 1.0e-10)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, obs_md_list, proper_chip, variability_cache, out_data): t_start_chunk = time.time() #print('processing %d' % len(chunk)) ct_first = 0 ct_at_all = 0 ct_tot = 0 n_t = len(filter_obs) n_obj = len(chunk) coadd_visits = {} coadd_visits['u'] = 6 coadd_visits['g'] = 8 coadd_visits['r'] = 18 coadd_visits['i'] = 18 coadd_visits['z'] = 16 coadd_visits['y'] = 16 # from the overview paper # table 2; take m5 row and add Delta m5 row # to get down to airmass 1.2 m5_single = {} m5_single['u'] = 23.57 m5_single['g'] = 24.65 m5_single['r'] = 24.21 m5_single['i'] = 23.79 m5_single['z'] = 23.21 m5_single['y'] = 22.31 gamma_coadd = {} for bp in 'ugrizy': gamma_coadd[bp] = None gamma_single = {} for bp in 'ugrizy': gamma_single[bp] = [None] * n_t dflux = np.zeros((n_obj, n_t), dtype=float) dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux) dummy_sed = Sed() lsst_bp = BandpassDict.loadTotalBandpassesFromFiles() flux_q = np.zeros((6, n_obj), dtype=float) flux_coadd = np.zeros((6, n_obj), dtype=float) mag_coadd = np.zeros((6, n_obj), dtype=float) snr_coadd = np.zeros((6, n_obj), dtype=float) snr_single = {} snr_single_mag_grid = np.arange(14.0, 30.0, 0.05) phot_params_single = PhotometricParameters(nexp=1, exptime=30.0) t_start_snr = time.time() photometry_mask = np.zeros((n_obj, n_t), dtype=bool) photometry_mask_1d = np.zeros(n_obj, dtype=bool) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) phot_params_coadd = PhotometricParameters(nexp=1, exptime=30.0 * coadd_visits[bp]) flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp]) dflux_sub = dflux[:, valid_obs[0]] assert dflux_sub.shape == (n_obj, len(valid_obs[0])) dflux_mean = np.mean(dflux_sub, axis=1) assert dflux_mean.shape == (n_obj, ) flux_coadd[i_bp] = flux_q[i_bp] + dflux_mean mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp]) (snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp], coadd_m5[bp], phot_params_coadd) (snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid, lsst_bp[bp], m5_single[bp], phot_params_single) #print('got all snr in %e' % (time.time()-t_start_snr)) t_start_obj = time.time() snr_arr = np.zeros((n_obj, n_t), dtype=float) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) if len(valid_obs[0]) == 0: continue n_bp = len(valid_obs[0]) dflux_bp = dflux[:, valid_obs[0]] flux0_arr = flux_q[i_bp] flux_tot = flux0_arr[:, None] + dflux_bp mag_tot = dummy_sed.magFromFlux(flux_tot) snr_single_val = np.interp(mag_tot, snr_single_mag_grid, snr_single[bp]) noise_coadd = flux_coadd[i_bp] / snr_coadd[i_bp] noise_single = flux_tot / snr_single_val noise = np.sqrt(noise_coadd[:, None]**2 + noise_single**2) dflux_thresh = 5.0 * noise dflux_bp = np.abs(dflux_bp) detected = (dflux_bp >= dflux_thresh) snr_arr[:, valid_obs[0]] = dflux_bp / noise for i_obj in range(n_obj): if detected[i_obj].any(): photometry_mask_1d[i_obj] = True photometry_mask[i_obj, valid_obs[0]] = detected[i_obj] t_before_chip = time.time() chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'], photometry_mask_1d, obs_md_list, filter_obs, proper_chip) duration = (time.time() - t_before_chip) / 3600.0 for i_obj in range(n_obj): if photometry_mask_1d[i_obj]: detected = photometry_mask[i_obj, :] & chip_mask[i_obj, :] if detected.any(): unq = chunk['simobjid'][i_obj] first_dex = np.where(detected)[0].min() out_data[unq] = (mjd_obs[first_dex], snr_arr[i_obj, first_dex], chunk['var_type'][i_obj]) if detected[0]: ct_first += 1 else: ct_at_all += 1
var_dict = json.loads(star['varParamStr']) lc = var_dict['p']['lc'] lc_int_dex[i_star] = lc_to_int[lc] for lc_id in np.unique(lc_int_dex): valid_stars = np.where(lc_int_dex == lc_id) if len(valid_stars[0]) == 0: continue local_flux_factor = flux_factor[valid_stars] local_dmag_max = np.zeros(len(valid_stars[0]), dtype=float) local_mag_min = 41.0 * np.ones(len(valid_stars[0]), dtype=float) for mag_name in mag_name_list: mag_0 = chunk['%smag' % mag_name][valid_stars] local_dust_factor = dust_factor[mag_name][valid_stars] flux_0 = dummy_sed.fluxFromMag(mag_0) dflux = dflux_dict[lc_id][ mag_name] * local_dust_factor * local_flux_factor total_flux = flux_0 + dflux mag = dummy_sed.magFromFlux(total_flux) dmag = 2.5 * np.log10(1.0 + dflux / flux_0) local_dmag_max = np.where(dmag > local_dmag_max, dmag, local_dmag_max) local_mag_min = np.where(mag < local_mag_min, mag, local_mag_min) dmag_max[valid_stars] = local_dmag_max mag_min[valid_stars] = local_mag_min ct_total += len(chunk) with open(out_name, 'a') as out_file:
params['t0'] = np.array([0.0] * n_steps) q_mags = {} for bp in 'ugrizy': q_mags[bp] = quiet_mag * np.ones(n_steps, dtype=float) for ebv_val in ebv_grid: ebv = np.array([ebv_val] * n_steps, dtype=float) dmag_base = mlt_model.applyMLTflaring(np.where( np.array(n_steps * [True])), params, t_max, parallax=parallax_grid, ebv=ebv, quiescent_mags=q_mags) mag_base = dmag_base + quiet_mag flux1 = dummy_sed.fluxFromMag(mag_base) flux0 = dummy_sed.fluxFromMag(quiet_mag) dflux = np.abs(flux1 - flux0) for ii, bp in enumerate('ugrizy'): dflux_name = '%s_ebv_%.2f_%s' % (lc_name, ebv_val, bp) assert dflux_name not in out_cache out_cache[dflux_name] = dflux[ii] out_name = 'data/mlt_dflux_lookup.h5' with h5py.File(out_name, 'w') as out_file: for name in out_cache: out_file.create_dataset(name, data=out_cache[name])
def test_avro_alert_generation_diff_dmag(self): """ Make sure everything works properly when the AlertDataGenerator and the AvroAlertGenerator have different dmag thresholds """ dmag_cutoff_sqlite = 0.005 dmag_cutoff_avro = 0.2 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') # assemble a dict of all of the alerts that need to be generated obshistid_list = [] for obs in self.obs_list: obshistid_list.append(obs.OpsimMetaData['obsHistID']) obshistid_max = max(obshistid_list) obshistid_bits = int(np.ceil(np.log(obshistid_max) / np.log(2.0))) true_alert_dict = {} obs_dict = {} ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) cat.camera = obs_lsst_phosim.PhosimMapper().camera for line in cat.iter_catalog(): if line[1] is None: continue dmag = line[2] mag = line[3] if (np.abs(dmag) > dmag_cutoff_avro and mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): alertId = (line[0] << obshistid_bits) + obshistid self.assertNotIn(alertId, true_alert_dict) true_alert_dict[alertId] = {} true_alert_dict[alertId]['chipName'] = line[1] true_alert_dict[alertId]['dmag'] = dmag true_alert_dict[alertId]['mag'] = mag true_alert_dict[alertId]['ra'] = np.degrees(line[4]) true_alert_dict[alertId]['decl'] = np.degrees(line[5]) true_alert_dict[alertId]['xPix'] = line[6] true_alert_dict[alertId]['yPix'] = line[7] elif np.abs(dmag) > dmag_cutoff_sqlite: ignored_sqlite += 1 self.assertGreater(len(true_alert_dict), 10) self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite # alerts were ignored by the more # stringent avro cut log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid( htmid, star_db, photometry_class=TestAlertsVarCat_avro, output_prefix='alert_test', output_dir=self.alert_data_output_dir, dmag_cutoff=dmag_cutoff_sqlite, log_file_name=log_file_name) obshistid_to_htmid = {} for htmid in alert_gen.htmid_list: for obs in alert_gen.obs_from_htmid(htmid): obshistid = obs.OpsimMetaData['obsHistID'] if obshistid not in obshistid_to_htmid: obshistid_to_htmid[obshistid] = [] obshistid_to_htmid[obshistid].append(htmid) avro_gen = AvroAlertGenerator() avro_gen.load_schema( os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) sql_prefix_list = ['alert_test'] out_prefix = 'test_avro' log_file_name = tempfile.mktemp(dir=self.avro_out_dir, prefix='test_avro', suffix='log.txt') for obshistid in obshistid_list: avro_gen.write_alerts(obshistid, self.alert_data_output_dir, sql_prefix_list, obshistid_to_htmid[obshistid], self.avro_out_dir, out_prefix, dmag_cutoff_avro, lock=None, log_file_name=log_file_name) list_of_avro_files = os.listdir(self.avro_out_dir) self.assertGreater(len(list_of_avro_files), 2) alert_ct = 0 dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() photParams = PhotometricParameters() diasourceId_set = set() for avro_file_name in list_of_avro_files: if avro_file_name.endswith('log.txt'): continue full_name = os.path.join(self.avro_out_dir, avro_file_name) with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: for alert in data_reader: alert_ct += 1 obshistid = alert['alertId'] >> 20 obs = obs_dict[obshistid] uniqueId = alert['diaObject']['diaObjectId'] true_alert_id = (uniqueId << obshistid_bits) + obshistid self.assertIn(true_alert_id, true_alert_dict) self.assertEqual(alert['l1dbId'], uniqueId) true_alert = true_alert_dict[true_alert_id] diaSource = alert['diaSource'] self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) true_q_mag = true_alert['mag'] - true_alert['dmag'] true_q_flux = dummy_sed.fluxFromMag(true_q_mag) true_dflux = true_tot_flux - true_q_flux self.assertAlmostEqual(diaSource['psFlux'] / true_dflux, 1.0, 6) self.assertAlmostEqual( diaSource['totFlux'] / true_tot_flux, 1.0, 6) self.assertAlmostEqual(diaSource['diffFlux'] / true_dflux, 1.0, 6) true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], obs.m5[obs.bandpass], photParams) true_q_snr, gamma = calcSNR_m5( true_q_mag, bp_dict[obs.bandpass], self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], photParams) true_tot_err = true_tot_flux / true_tot_snr true_q_err = true_q_flux / true_q_snr true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) self.assertAlmostEqual( diaSource['snr'] / np.abs(true_dflux / true_diff_err), 1.0, 6) self.assertAlmostEqual( diaSource['totFluxErr'] / true_tot_err, 1.0, 6) self.assertAlmostEqual( diaSource['diffFluxErr'] / true_diff_err, 1.0, 6) chipnum = int(true_alert['chipName'].replace( 'R', '').replace('S', '').replace(',', '').replace( ':', '').replace(' ', '')) true_ccdid = (chipnum * 10**7) + obshistid self.assertEqual(true_ccdid, diaSource['ccdVisitId']) self.assertEqual(uniqueId, diaSource['diaObjectId']) self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) diasourceId_set.add(diaSource['diaSourceId']) diaObject = alert['diaObject'] obj_dex = (uniqueId // 1024) - 1 self.assertAlmostEqual( 0.001 * diaObject['pmRa'] / self.pmra_truth[obj_dex], 1.0, 5) self.assertAlmostEqual( 0.001 * diaObject['pmDecl'] / self.pmdec_truth[obj_dex], 1.0, 5) self.assertAlmostEqual( 0.001 * diaObject['parallax'] / self.px_truth[obj_dex], 1.0, 5) (true_ra_base, true_dec_base) = applyProperMotion( self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) self.assertEqual(alert_ct, len(true_alert_dict))
# in cm) flux_factor = params.effarea/(4.0*np.pi*np.power(dd*3.08576e18, 2)) types_simulated = [] du = [] dg = [] dr = [] di = [] dz = [] dy = [] from mdwarf_utils import light_curve_from_class from lsst.sims.photUtils import Sed ss = Sed() base_u = ss.fluxFromMag(data['umag']) base_g = ss.fluxFromMag(data['gmag']) base_r = ss.fluxFromMag(data['rmag']) base_i = ss.fluxFromMag(data['imag']) base_z = ss.fluxFromMag(data['zmag']) base_y = ss.fluxFromMag(data['ymag']) for i_star in range(len(data)): thermo_class = type_dict[prob_vec[i_star]] if thermo_class == 'late': activity = 'active' else: activity = activity_level[i_star] flare_type = '%s_%s' % (thermo_class, activity) if flare_type in types_simulated:
def process_agn_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, m5_single, obs_md_list, proper_chip, out_data, lock): t_start_chunk = time.time() #print('processing %d' % len(chunk)) ct_first = 0 ct_at_all = 0 ct_tot = 0 n_t = len(filter_obs) n_obj = len(chunk) agn_model = ExtraGalacticVariabilityModels() dust_model = EBVbase() with h5py.File('data/ebv_grid.h5', 'r') as in_file: ebv_grid = in_file['ebv_grid'].value extinction_grid = in_file['extinction_grid'].value coadd_visits = {} coadd_visits['u'] = 6 coadd_visits['g'] = 8 coadd_visits['r'] = 18 coadd_visits['i'] = 18 coadd_visits['z'] = 16 coadd_visits['y'] = 16 gamma_coadd = {} for bp in 'ugrizy': gamma_coadd[bp] = None gamma_single = {} for bp in 'ugrizy': gamma_single[bp] = [None]*n_t params = {} params['agn_sfu'] = chunk['agn_sfu'] params['agn_sfg'] = chunk['agn_sfg'] params['agn_sfr'] = chunk['agn_sfr'] params['agn_sfi'] = chunk['agn_sfi'] params['agn_sfz'] = chunk['agn_sfz'] params['agn_sfy'] = chunk['agn_sfy'] params['agn_tau'] = chunk['agn_tau'] params['seed'] = chunk['id']+1 ebv = dust_model.calculateEbv(equatorialCoordinates=np.array([np.radians(chunk['ra']), np.radians(chunk['dec'])]), interp=True) for i_bp, bp in enumerate('ugrizy'): extinction_values = np.interp(ebv, ebv_grid, extinction_grid[i_bp]) chunk['%s_ab' % bp] += extinction_values chunk['AGNLSST%s' % bp] += extinction_values dmag = agn_model.applyAgn(np.where(np.array([True]*len(chunk))), params, mjd_obs, redshift=chunk['redshift']) dmag_mean = np.mean(dmag, axis=2) assert dmag_mean.shape == (6,n_obj) dummy_sed = Sed() lsst_bp = BandpassDict.loadTotalBandpassesFromFiles() flux_gal = np.zeros((6,n_obj), dtype=float) flux_agn_q = np.zeros((6,n_obj), dtype=float) flux_coadd = np.zeros((6,n_obj), dtype=float) mag_coadd = np.zeros((6,n_obj), dtype=float) snr_coadd = np.zeros((6,n_obj), dtype=float) snr_single = {} snr_single_mag_grid = np.arange(14.0, 30.0, 0.05) phot_params_single = PhotometricParameters(nexp=1, exptime=30.0) t_start_snr = time.time() for i_bp, bp in enumerate('ugrizy'): phot_params_coadd = PhotometricParameters(nexp=1, exptime=30.0*coadd_visits[bp]) flux_gal[i_bp] = dummy_sed.fluxFromMag(chunk['%s_ab' % bp]) flux_agn_q[i_bp] = dummy_sed.fluxFromMag(chunk['AGNLSST%s' % bp] + dmag_mean[i_bp,:]) flux_coadd[i_bp] = flux_gal[i_bp]+flux_agn_q[i_bp] mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp]) (snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp], coadd_m5[bp], phot_params_coadd) (snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid, lsst_bp[bp], m5_single[bp], phot_params_single) #print('got all snr in %e' % (time.time()-t_start_snr)) t_start_obj = time.time() photometry_mask = np.zeros((n_obj, n_t), dtype=bool) photometry_mask_1d = np.zeros(n_obj, dtype=bool) snr_arr = np.zeros((n_obj, n_t), dtype=float) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs==i_bp) n_bp = len(valid_obs[0]) if n_bp == 0: continue mag0_arr = chunk['AGNLSST%s' % bp] dmag_bp = dmag[i_bp][:,valid_obs[0]] assert dmag_bp.shape == (n_obj, n_bp) agn_flux_tot = dummy_sed.fluxFromMag(mag0_arr[:,None]+dmag_bp) q_flux = flux_agn_q[i_bp] agn_dflux = np.abs(agn_flux_tot-q_flux[:,None]) flux_tot = flux_gal[i_bp][:, None] + agn_flux_tot assert flux_tot.shape == (n_obj, n_bp) mag_tot = dummy_sed.magFromFlux(flux_tot) snr_single_val = np.interp(mag_tot, snr_single_mag_grid, snr_single[bp]) noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp] noise_single = flux_tot/snr_single_val noise = np.sqrt(noise_coadd[:,None]**2 + noise_single**2) dflux_thresh = 5.0*noise detected = (agn_dflux>=dflux_thresh) assert detected.shape == (n_obj, n_bp) snr_arr[:,valid_obs[0]] = agn_dflux/noise for i_obj in range(n_obj): if detected[i_obj].any(): photometry_mask_1d[i_obj] = True photometry_mask[i_obj, valid_obs[0]] = detected[i_obj] t_before_chip = time.time() chip_mask = apply_focal_plane(chunk['ra'], chunk['dec'], photometry_mask_1d, obs_md_list, filter_obs, proper_chip) duration = (time.time()-t_before_chip)/3600.0 unq_out = -1*np.ones(n_obj, dtype=int) mjd_out = -1.0*np.ones(n_obj, dtype=float) snr_out = -1.0*np.ones(n_obj, dtype=float) for i_obj in range(n_obj): if photometry_mask_1d[i_obj]: detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:] if detected.any(): unq_out[i_obj] = chunk['galtileid'][i_obj] first_dex = np.where(detected)[0].min() mjd_out[i_obj] = mjd_obs[first_dex] snr_out[i_obj] = snr_arr[i_obj, first_dex] valid = np.where(unq_out>=0) unq_out = unq_out[valid] mjd_out = mjd_out[valid] snr_out = snr_out[valid] with lock: existing_keys = list(out_data.keys()) if len(existing_keys) == 0: key_val = 0 else: key_val = max(existing_keys) while key_val in existing_keys: key_val += 1 out_data[key_val] = (None, None, None) out_data[key_val] = (unq_out, mjd_out, snr_out)
def applyMLTflaring(self, valid_dexes, params, expmjd, parallax=None, ebv=None, quiescent_mags=None): """ parallax, ebv, and quiescent_mags are optional kwargs for use if you are calling this method outside the context of an InstanceCatalog (presumably with a numpy array of expmjd) parallax is the parallax of your objects in radians ebv is the E(B-V) value for your objects quiescent_mags is a dict keyed on ('u', 'g', 'r', 'i', 'z', 'y') with the quiescent magnitudes of the objects """ if parallax is None: parallax = self.column_by_name('parallax') if ebv is None: ebv = self.column_by_name('ebv') global _MLT_LC_NPZ global _MLT_LC_NPZ_NAME global _MLT_LC_TIME_CACHE global _MLT_LC_FLUX_CACHE # this needs to occur before loading the MLT light curve cache, # just in case the user wants to override the light curve cache # file by hand before generating the catalog if len(params) == 0: return numpy.array([[],[],[],[],[],[]]) if quiescent_mags is None: quiescent_mags = {} for mag_name in ('u', 'g', 'r', 'i', 'z', 'y'): if ('lsst_%s' % mag_name in self._actually_calculated_columns or 'delta_lsst_%s' % mag_name in self._actually_calculated_columns): quiescent_mags[mag_name] = self.column_by_name('quiescent_lsst_%s' % mag_name) if not hasattr(self, 'photParams'): raise RuntimeError("To apply MLT dwarf flaring, your " "InstanceCatalog must have a member variable " "photParams which is an instantiation of the " "class PhotometricParameters, which can be " "imported from lsst.sims.photUtils. " "This is so that your InstanceCatalog has " "knowledge of the effective area of the LSST " "mirror.") if _MLT_LC_NPZ is None or _MLT_LC_NPZ_NAME != self._mlt_lc_file or _MLT_LC_NPZ.fid is None: if not os.path.exists(self._mlt_lc_file): catutils_scripts = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts') raise RuntimeError("The MLT flaring light curve file:\n" + "\n%s\n" % self._mlt_lc_file + "\ndoes not exist." +"\n\n" + "Go into %s " % catutils_scripts + "and run get_mdwarf_flares.sh " + "to get the data") _MLT_LC_NPZ = numpy.load(self._mlt_lc_file) sims_clean_up.targets.append(_MLT_LC_NPZ) _MLT_LC_NPZ_NAME = self._mlt_lc_file _MLT_LC_TIME_CACHE = {} _MLT_LC_FLUX_CACHE = {} if not hasattr(self, '_mlt_dust_lookup'): # Construct a look-up table to determine the factor # by which to multiply the flares' flux to account for # dust as a function of E(B-V). Recall that we are # modeling all MLT flares as 9000K blackbodies. if not hasattr(self, 'lsstBandpassDict'): raise RuntimeError('You are asking for MLT dwarf flaring ' 'magnitudes in a catalog that has not ' 'defined lsstBandpassDict. The MLT ' 'flaring magnitudes model does not know ' 'how to apply dust extinction to the ' 'flares without the member variable ' 'lsstBandpassDict being defined.') ebv_grid = numpy.arange(0.0, 7.01, 0.01) bb_wavelen = numpy.arange(200.0, 1500.0, 0.1) hc_over_k = 1.4387e7 # nm*K temp = 9000.0 # black body temperature in Kelvin exp_arg = hc_over_k/(temp*bb_wavelen) exp_term = 1.0/(numpy.exp(exp_arg) - 1.0) ln_exp_term = numpy.log(exp_term) # Blackbody f_lambda function; # discard normalizing factors; we only care about finding the # ratio of fluxes between the case with dust extinction and # the case without dust extinction log_bb_flambda = -5.0*numpy.log(bb_wavelen) + ln_exp_term bb_flambda = numpy.exp(log_bb_flambda) bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda) base_fluxes = self.lsstBandpassDict.fluxListForSed(bb_sed) a_x, b_x = bb_sed.setupCCMab() self._mlt_dust_lookup = {} self._mlt_dust_lookup['ebv'] = ebv_grid list_of_bp = self.lsstBandpassDict.keys() for bp in list_of_bp: self._mlt_dust_lookup[bp] = numpy.zeros(len(ebv_grid)) for iebv, ebv_val in enumerate(ebv_grid): wv, fl = bb_sed.addCCMDust(a_x, b_x, ebv=ebv_val, wavelen=bb_wavelen, flambda=bb_flambda) dusty_bb = Sed(wavelen=wv, flambda=fl) dusty_fluxes = self.lsstBandpassDict.fluxListForSed(dusty_bb) for ibp, bp in enumerate(list_of_bp): self._mlt_dust_lookup[bp][iebv] = dusty_fluxes[ibp]/base_fluxes[ibp] # get the distance to each star in parsecs _au_to_parsec = 1.0/206265.0 dd = _au_to_parsec/parallax # get the area of the sphere through which the star's energy # is radiating to get to us (in cm^2) _cm_per_parsec = 3.08576e16 sphere_area = 4.0*numpy.pi*numpy.power(dd*_cm_per_parsec, 2) flux_factor = self.photParams.effarea/sphere_area if isinstance(expmjd, numbers.Number): dMags = numpy.zeros((6, self.num_variable_obj(params))) else: dMags = numpy.zeros((6, self.num_variable_obj(params), len(expmjd))) mag_name_tuple = ('u', 'g', 'r', 'i', 'z', 'y') base_fluxes = {} base_mags = {} ss = Sed() for mag_name in mag_name_tuple: if ('lsst_%s' % mag_name in self._actually_calculated_columns or 'delta_lsst_%s' % mag_name in self._actually_calculated_columns): mm = quiescent_mags[mag_name] base_mags[mag_name] = mm base_fluxes[mag_name] = ss.fluxFromMag(mm) lc_name_arr = params['lc'].astype(str) lc_names_unique = numpy.unique(lc_name_arr) for lc_name in lc_names_unique: if 'None' in lc_name: continue use_this_lc = numpy.where(numpy.char.find(lc_name_arr, lc_name)==0) lc_name = lc_name.replace('.txt', '') # 2017 May 1 # There isn't supposed to be a 'late_inactive' light curve. # Unfortunately, I (Scott Daniel) assigned 'late_inactive' # light curves to some of the stars on our database. Rather # than fix the database table (which will take about a week of # compute time), I am going to fix the problem here by mapping # 'late_inactive' into 'late_active'. if 'late' in lc_name: lc_name = lc_name.replace('in', '') if lc_name in _MLT_LC_TIME_CACHE: raw_time_arr = _MLT_LC_TIME_CACHE[lc_name] else: raw_time_arr = _MLT_LC_NPZ['%s_time' % lc_name] _MLT_LC_TIME_CACHE[lc_name] = raw_time_arr time_arr = self._survey_start + raw_time_arr dt = time_arr.max() - time_arr.min() if isinstance(expmjd, numbers.Number): t_interp = (expmjd + params['t0'][use_this_lc]).astype(float) else: n_obj = len(use_this_lc[0]) n_time = len(expmjd) t_interp = numpy.ones(shape=(n_obj, n_time))*expmjd t_interp += numpy.array([[tt]*n_time for tt in params['t0'][use_this_lc].astype(float)]) while t_interp.max() > time_arr.max(): bad_dexes = numpy.where(t_interp>time_arr.max()) t_interp[bad_dexes] -= dt for i_mag, mag_name in enumerate(mag_name_tuple): if ('lsst_%s' % mag_name in self._actually_calculated_columns or 'delta_lsst_%s' % mag_name in self._actually_calculated_columns): flux_name = '%s_%s' % (lc_name, mag_name) if flux_name in _MLT_LC_FLUX_CACHE: flux_arr = _MLT_LC_FLUX_CACHE[flux_name] else: flux_arr = _MLT_LC_NPZ[flux_name] _MLT_LC_FLUX_CACHE[flux_name] = flux_arr dflux = numpy.interp(t_interp, time_arr, flux_arr) if isinstance(expmjd, numbers.Number): dflux *= flux_factor[use_this_lc] else: dflux *= numpy.array([flux_factor[use_this_lc]]*n_time).transpose() dust_factor = numpy.interp(ebv[use_this_lc], self._mlt_dust_lookup['ebv'], self._mlt_dust_lookup[mag_name]) if not isinstance(expmjd, numbers.Number): dust_factor = numpy.array([dust_factor]*n_time).transpose() dflux *= dust_factor if isinstance(expmjd, numbers.Number): local_base_fluxes = base_fluxes[mag_name][use_this_lc] local_base_mags = base_mags[mag_name][use_this_lc] else: local_base_fluxes = numpy.array([base_fluxes[mag_name][use_this_lc]]*n_time).transpose() local_base_mags = numpy.array([base_mags[mag_name][use_this_lc]]*n_time).transpose() dMags[i_mag][use_this_lc] = (ss.magFromFlux(local_base_fluxes + dflux) - local_base_mags) return dMags
def test_avro_alert_generation_diff_dmag(self): """ Make sure everything works properly when the AlertDataGenerator and the AvroAlertGenerator have different dmag thresholds """ dmag_cutoff_sqlite = 0.005 dmag_cutoff_avro = 0.2 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5} star_db = StarAlertTestDBObj_avro(database=self.star_db_name, driver='sqlite') # assemble a dict of all of the alerts that need to be generated obshistid_list = [] for obs in self.obs_list: obshistid_list.append(obs.OpsimMetaData['obsHistID']) obshistid_max = max(obshistid_list) obshistid_bits = int(np.ceil(np.log(obshistid_max)/np.log(2.0))) true_alert_dict = {} obs_dict = {} ignored_sqlite = 0 # count number of alerts written to sqlite, but not avro for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] cat = TestAlertsTruthCat_avro(star_db, obs_metadata=obs) cat.camera = lsst_camera() for line in cat.iter_catalog(): if line[1] is None: continue dmag = line[2] mag = line[3] if (np.abs(dmag) > dmag_cutoff_avro and mag <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]): alertId = (line[0] << obshistid_bits) + obshistid self.assertNotIn(alertId, true_alert_dict) true_alert_dict[alertId] = {} true_alert_dict[alertId]['chipName'] = line[1] true_alert_dict[alertId]['dmag'] = dmag true_alert_dict[alertId]['mag'] = mag true_alert_dict[alertId]['ra'] = np.degrees(line[4]) true_alert_dict[alertId]['decl'] = np.degrees(line[5]) true_alert_dict[alertId]['xPix'] = line[6] true_alert_dict[alertId]['yPix'] = line[7] elif np.abs(dmag) > dmag_cutoff_sqlite: ignored_sqlite += 1 self.assertGreater(len(true_alert_dict), 10) self.assertGreater(ignored_sqlite, 50) # just make sure that some sqlite # alerts were ignored by the more # stringent avro cut log_file_name = tempfile.mktemp(dir=self.alert_data_output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid(htmid, star_db, photometry_class=TestAlertsVarCat_avro, output_prefix='alert_test', output_dir=self.alert_data_output_dir, dmag_cutoff=dmag_cutoff_sqlite, log_file_name=log_file_name) obshistid_to_htmid = {} for htmid in alert_gen.htmid_list: for obs in alert_gen.obs_from_htmid(htmid): obshistid = obs.OpsimMetaData['obsHistID'] if obshistid not in obshistid_to_htmid: obshistid_to_htmid[obshistid] = [] obshistid_to_htmid[obshistid].append(htmid) avro_gen = AvroAlertGenerator() avro_gen.load_schema(os.path.join(getPackageDir('sims_catUtils'), 'tests', 'testData', 'avroSchema')) sql_prefix_list = ['alert_test'] out_prefix = 'test_avro' log_file_name = tempfile.mktemp(dir=self.avro_out_dir, prefix='test_avro', suffix='log.txt') for obshistid in obshistid_list: avro_gen.write_alerts(obshistid, self.alert_data_output_dir, sql_prefix_list, obshistid_to_htmid[obshistid], self.avro_out_dir, out_prefix, dmag_cutoff_avro, lock=None, log_file_name=log_file_name) list_of_avro_files = os.listdir(self.avro_out_dir) self.assertGreater(len(list_of_avro_files), 2) alert_ct = 0 dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() photParams = PhotometricParameters() diasourceId_set = set() for avro_file_name in list_of_avro_files: if avro_file_name.endswith('log.txt'): continue full_name = os.path.join(self.avro_out_dir, avro_file_name) with DataFileReader(open(full_name, 'rb'), DatumReader()) as data_reader: for alert in data_reader: alert_ct += 1 obshistid = alert['alertId'] >> 20 obs = obs_dict[obshistid] uniqueId = alert['diaObject']['diaObjectId'] true_alert_id = (uniqueId << obshistid_bits) + obshistid self.assertIn(true_alert_id, true_alert_dict) self.assertEqual(alert['l1dbId'], uniqueId) true_alert = true_alert_dict[true_alert_id] diaSource = alert['diaSource'] self.assertAlmostEqual(diaSource['ra'], true_alert['ra'], 10) self.assertAlmostEqual(diaSource['decl'], true_alert['decl'], 10) self.assertAlmostEqual(diaSource['x'], true_alert['xPix'], 3) self.assertAlmostEqual(diaSource['y'], true_alert['yPix'], 3) self.assertAlmostEqual(diaSource['midPointTai'], obs.mjd.TAI, 4) true_tot_flux = dummy_sed.fluxFromMag(true_alert['mag']) true_q_mag = true_alert['mag'] - true_alert['dmag'] true_q_flux = dummy_sed.fluxFromMag(true_q_mag) true_dflux = true_tot_flux - true_q_flux self.assertAlmostEqual(diaSource['psFlux']/true_dflux, 1.0, 6) self.assertAlmostEqual(diaSource['totFlux']/true_tot_flux, 1.0, 6) self.assertAlmostEqual(diaSource['diffFlux']/true_dflux, 1.0, 6) true_tot_snr, gamma = calcSNR_m5(true_alert['mag'], bp_dict[obs.bandpass], obs.m5[obs.bandpass], photParams) true_q_snr, gamma = calcSNR_m5(true_q_mag, bp_dict[obs.bandpass], self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]], photParams) true_tot_err = true_tot_flux/true_tot_snr true_q_err = true_q_flux/true_q_snr true_diff_err = np.sqrt(true_tot_err**2 + true_q_err**2) self.assertAlmostEqual(diaSource['snr']/np.abs(true_dflux/true_diff_err), 1.0, 6) self.assertAlmostEqual(diaSource['totFluxErr']/true_tot_err, 1.0, 6) self.assertAlmostEqual(diaSource['diffFluxErr']/true_diff_err, 1.0, 6) chipnum = int(true_alert['chipName'].replace('R', '').replace('S', ''). replace(',', '').replace(':', '').replace(' ', '')) true_ccdid = (chipnum*10**7)+obshistid self.assertEqual(true_ccdid, diaSource['ccdVisitId']) self.assertEqual(uniqueId, diaSource['diaObjectId']) self.assertNotIn(diaSource['diaSourceId'], diasourceId_set) diasourceId_set.add(diaSource['diaSourceId']) diaObject = alert['diaObject'] obj_dex = (uniqueId//1024) - 1 self.assertAlmostEqual(0.001*diaObject['pmRa']/self.pmra_truth[obj_dex], 1.0, 5) self.assertAlmostEqual(0.001*diaObject['pmDecl']/self.pmdec_truth[obj_dex], 1.0, 5) self.assertAlmostEqual(0.001*diaObject['parallax']/self.px_truth[obj_dex], 1.0, 5) (true_ra_base, true_dec_base) = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=ModifiedJulianDate(TAI=diaObject['radecTai'])) self.assertAlmostEqual(true_ra_base, diaObject['ra'], 7) self.assertAlmostEqual(true_dec_base, diaObject['decl'], 7) self.assertEqual(alert_ct, len(true_alert_dict))