def test_mags_vs_flux(self):
"""
Verify that the relationship between Sed.calcMag() and Sed.calcFlux()
is as expected
"""
wavelen = np.arange(100.0, 1500.0, 1.0)
flambda = np.exp(-0.5*np.power((wavelen-500.0)/100.0,2))
sb = (wavelen-100.0)/1400.0
ss = Sed(wavelen=wavelen, flambda=flambda)
bp = Bandpass(wavelen=wavelen, sb=sb)
mag = ss.calcMag(bp)
flux = ss.calcFlux(bp)
self.assertAlmostEqual(ss.magFromFlux(flux)/mag, 1.0, 10)
self.assertAlmostEqual(ss.fluxFromMag(mag)/flux, 1.0, 10)
def test_mags_vs_flux(self):
"""
Verify that the relationship between Sed.calcMag() and Sed.calcFlux()
is as expected
"""
wavelen = np.arange(100.0, 1500.0, 1.0)
flambda = np.exp(-0.5 * np.power((wavelen - 500.0) / 100.0, 2))
sb = (wavelen - 100.0) / 1400.0
ss = Sed(wavelen=wavelen, flambda=flambda)
bp = Bandpass(wavelen=wavelen, sb=sb)
mag = ss.calcMag(bp)
flux = ss.calcFlux(bp)
self.assertAlmostEqual(ss.magFromFlux(flux) / mag, 1.0, 10)
self.assertAlmostEqual(ss.fluxFromMag(mag) / flux, 1.0, 10)
def calculate_mags(galaxy_list, out_dict):
"""
Calculate the total (bulge+disk+agn) magnitudes of galaxies
Parameters
----------
galaxy_list is a list of tuples. The tuples are of the form
(bulge.sed_name, bulge.magNorm, disk.sed_name, disk.magNorm
agn.sed_name, agn.magNorm)
out_dict is a Multiprocessing.Manager.dict object that will
store the results of this calculation
"""
i_process = mp.current_process().pid
bulge_fluxes = np.zeros((len(galaxy_list), 6), dtype=float)
disk_fluxes = np.zeros((len(galaxy_list), 6), dtype=float)
magnification = np.array(
[1.0 / ((1.0 - g[14])**2 - g[12]**2 - g[13]**2) for g in galaxy_list])
assert magnification.min() >= 0.0
assert len(np.where(np.isfinite(magnification))[0]) == len(magnification)
for i_gal, galaxy in enumerate(galaxy_list):
if galaxy[0] is not None and galaxy[1] is not None:
bulge_fluxes[i_gal] = _fluxes(galaxy[0], galaxy[1], galaxy[6])
if galaxy[2] is not None and galaxy[3] is not None:
disk_fluxes[i_gal] = _fluxes(galaxy[2], galaxy[3], galaxy[6])
tot_fluxes = bulge_fluxes + disk_fluxes
for i_filter in range(6):
tot_fluxes[:, i_filter] *= magnification
dummy_sed = Sed()
valid = np.where(tot_fluxes > 0.0)
valid_mags = dummy_sed.magFromFlux(tot_fluxes[valid])
out_mags = np.NaN * np.ones((len(galaxy_list), 6), dtype=float)
out_mags[valid] = valid_mags
out_dict[i_process] = out_mags
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoords, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
camera = obs_lsst_phosim.PhosimMapper().camera
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
camera=self.camera)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
def applyMLTflaring(self, valid_dexes, params, expmjd,
parallax=None, ebv=None, quiescent_mags=None):
"""
parallax, ebv, and quiescent_mags are optional kwargs for use if you are
calling this method outside the context of an InstanceCatalog (presumably
with a numpy array of expmjd)
parallax is the parallax of your objects in radians
ebv is the E(B-V) value for your objects
quiescent_mags is a dict keyed on ('u', 'g', 'r', 'i', 'z', 'y')
with the quiescent magnitudes of the objects
"""
if parallax is None:
parallax = self.column_by_name('parallax')
if ebv is None:
ebv = self.column_by_name('ebv')
global _MLT_LC_NPZ
global _MLT_LC_NPZ_NAME
global _MLT_LC_TIME_CACHE
global _MLT_LC_FLUX_CACHE
# this needs to occur before loading the MLT light curve cache,
# just in case the user wants to override the light curve cache
# file by hand before generating the catalog
if len(params) == 0:
return numpy.array([[],[],[],[],[],[]])
if quiescent_mags is None:
quiescent_mags = {}
for mag_name in ('u', 'g', 'r', 'i', 'z', 'y'):
if ('lsst_%s' % mag_name in self._actually_calculated_columns or
'delta_lsst_%s' % mag_name in self._actually_calculated_columns):
quiescent_mags[mag_name] = self.column_by_name('quiescent_lsst_%s' % mag_name)
if not hasattr(self, 'photParams'):
raise RuntimeError("To apply MLT dwarf flaring, your "
"InstanceCatalog must have a member variable "
"photParams which is an instantiation of the "
"class PhotometricParameters, which can be "
"imported from lsst.sims.photUtils. "
"This is so that your InstanceCatalog has "
"knowledge of the effective area of the LSST "
"mirror.")
if _MLT_LC_NPZ is None or _MLT_LC_NPZ_NAME != self._mlt_lc_file or _MLT_LC_NPZ.fid is None:
if not os.path.exists(self._mlt_lc_file):
catutils_scripts = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts')
raise RuntimeError("The MLT flaring light curve file:\n"
+ "\n%s\n" % self._mlt_lc_file
+ "\ndoes not exist."
+"\n\n"
+ "Go into %s " % catutils_scripts
+ "and run get_mdwarf_flares.sh "
+ "to get the data")
_MLT_LC_NPZ = numpy.load(self._mlt_lc_file)
sims_clean_up.targets.append(_MLT_LC_NPZ)
_MLT_LC_NPZ_NAME = self._mlt_lc_file
_MLT_LC_TIME_CACHE = {}
_MLT_LC_FLUX_CACHE = {}
if not hasattr(self, '_mlt_dust_lookup'):
# Construct a look-up table to determine the factor
# by which to multiply the flares' flux to account for
# dust as a function of E(B-V). Recall that we are
# modeling all MLT flares as 9000K blackbodies.
if not hasattr(self, 'lsstBandpassDict'):
raise RuntimeError('You are asking for MLT dwarf flaring '
'magnitudes in a catalog that has not '
'defined lsstBandpassDict. The MLT '
'flaring magnitudes model does not know '
'how to apply dust extinction to the '
'flares without the member variable '
'lsstBandpassDict being defined.')
ebv_grid = numpy.arange(0.0, 7.01, 0.01)
bb_wavelen = numpy.arange(200.0, 1500.0, 0.1)
hc_over_k = 1.4387e7 # nm*K
temp = 9000.0 # black body temperature in Kelvin
exp_arg = hc_over_k/(temp*bb_wavelen)
exp_term = 1.0/(numpy.exp(exp_arg) - 1.0)
ln_exp_term = numpy.log(exp_term)
# Blackbody f_lambda function;
# discard normalizing factors; we only care about finding the
# ratio of fluxes between the case with dust extinction and
# the case without dust extinction
log_bb_flambda = -5.0*numpy.log(bb_wavelen) + ln_exp_term
bb_flambda = numpy.exp(log_bb_flambda)
bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda)
base_fluxes = self.lsstBandpassDict.fluxListForSed(bb_sed)
a_x, b_x = bb_sed.setupCCMab()
self._mlt_dust_lookup = {}
self._mlt_dust_lookup['ebv'] = ebv_grid
list_of_bp = self.lsstBandpassDict.keys()
for bp in list_of_bp:
self._mlt_dust_lookup[bp] = numpy.zeros(len(ebv_grid))
for iebv, ebv_val in enumerate(ebv_grid):
wv, fl = bb_sed.addCCMDust(a_x, b_x,
ebv=ebv_val,
wavelen=bb_wavelen,
flambda=bb_flambda)
dusty_bb = Sed(wavelen=wv, flambda=fl)
dusty_fluxes = self.lsstBandpassDict.fluxListForSed(dusty_bb)
for ibp, bp in enumerate(list_of_bp):
self._mlt_dust_lookup[bp][iebv] = dusty_fluxes[ibp]/base_fluxes[ibp]
# get the distance to each star in parsecs
_au_to_parsec = 1.0/206265.0
dd = _au_to_parsec/parallax
# get the area of the sphere through which the star's energy
# is radiating to get to us (in cm^2)
_cm_per_parsec = 3.08576e16
sphere_area = 4.0*numpy.pi*numpy.power(dd*_cm_per_parsec, 2)
flux_factor = self.photParams.effarea/sphere_area
if isinstance(expmjd, numbers.Number):
dMags = numpy.zeros((6, self.num_variable_obj(params)))
else:
dMags = numpy.zeros((6, self.num_variable_obj(params), len(expmjd)))
mag_name_tuple = ('u', 'g', 'r', 'i', 'z', 'y')
base_fluxes = {}
base_mags = {}
ss = Sed()
for mag_name in mag_name_tuple:
if ('lsst_%s' % mag_name in self._actually_calculated_columns or
'delta_lsst_%s' % mag_name in self._actually_calculated_columns):
mm = quiescent_mags[mag_name]
base_mags[mag_name] = mm
base_fluxes[mag_name] = ss.fluxFromMag(mm)
lc_name_arr = params['lc'].astype(str)
lc_names_unique = numpy.unique(lc_name_arr)
for lc_name in lc_names_unique:
if 'None' in lc_name:
continue
use_this_lc = numpy.where(numpy.char.find(lc_name_arr, lc_name)==0)
lc_name = lc_name.replace('.txt', '')
# 2017 May 1
# There isn't supposed to be a 'late_inactive' light curve.
# Unfortunately, I (Scott Daniel) assigned 'late_inactive'
# light curves to some of the stars on our database. Rather
# than fix the database table (which will take about a week of
# compute time), I am going to fix the problem here by mapping
# 'late_inactive' into 'late_active'.
if 'late' in lc_name:
lc_name = lc_name.replace('in', '')
if lc_name in _MLT_LC_TIME_CACHE:
raw_time_arr = _MLT_LC_TIME_CACHE[lc_name]
else:
raw_time_arr = _MLT_LC_NPZ['%s_time' % lc_name]
_MLT_LC_TIME_CACHE[lc_name] = raw_time_arr
time_arr = self._survey_start + raw_time_arr
dt = time_arr.max() - time_arr.min()
if isinstance(expmjd, numbers.Number):
t_interp = (expmjd + params['t0'][use_this_lc]).astype(float)
else:
n_obj = len(use_this_lc[0])
n_time = len(expmjd)
t_interp = numpy.ones(shape=(n_obj, n_time))*expmjd
t_interp += numpy.array([[tt]*n_time for tt in params['t0'][use_this_lc].astype(float)])
while t_interp.max() > time_arr.max():
bad_dexes = numpy.where(t_interp>time_arr.max())
t_interp[bad_dexes] -= dt
for i_mag, mag_name in enumerate(mag_name_tuple):
if ('lsst_%s' % mag_name in self._actually_calculated_columns or
'delta_lsst_%s' % mag_name in self._actually_calculated_columns):
flux_name = '%s_%s' % (lc_name, mag_name)
if flux_name in _MLT_LC_FLUX_CACHE:
flux_arr = _MLT_LC_FLUX_CACHE[flux_name]
else:
flux_arr = _MLT_LC_NPZ[flux_name]
_MLT_LC_FLUX_CACHE[flux_name] = flux_arr
dflux = numpy.interp(t_interp, time_arr, flux_arr)
if isinstance(expmjd, numbers.Number):
dflux *= flux_factor[use_this_lc]
else:
dflux *= numpy.array([flux_factor[use_this_lc]]*n_time).transpose()
dust_factor = numpy.interp(ebv[use_this_lc],
self._mlt_dust_lookup['ebv'],
self._mlt_dust_lookup[mag_name])
if not isinstance(expmjd, numbers.Number):
dust_factor = numpy.array([dust_factor]*n_time).transpose()
dflux *= dust_factor
if isinstance(expmjd, numbers.Number):
local_base_fluxes = base_fluxes[mag_name][use_this_lc]
local_base_mags = base_mags[mag_name][use_this_lc]
else:
local_base_fluxes = numpy.array([base_fluxes[mag_name][use_this_lc]]*n_time).transpose()
local_base_mags = numpy.array([base_mags[mag_name][use_this_lc]]*n_time).transpose()
dMags[i_mag][use_this_lc] = (ss.magFromFlux(local_base_fluxes + dflux)
- local_base_mags)
return dMags
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs,
coadd_m5, m5_single, obs_md_list, proper_chip,
variability_cache, out_data, lock):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None]*n_t
dflux = np.zeros((n_obj,n_t), dtype=float)
dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_q = np.zeros((6,n_obj), dtype=float)
flux_coadd = np.zeros((6,n_obj), dtype=float)
mag_coadd = np.zeros((6,n_obj), dtype=float)
snr_coadd = np.zeros((6,n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1,
exptime=30.0)
t_start_snr = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0*coadd_visits[bp])
flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp])
dflux_sub = dflux[:,valid_obs[0]]
assert dflux_sub.shape == (n_obj, len(valid_obs[0]))
dflux_mean = np.mean(dflux_sub, axis=1)
assert dflux_mean.shape==(n_obj,)
flux_coadd[i_bp] = flux_q[i_bp]+dflux_mean
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp],
gamma) = SNR.calcSNR_m5(mag_coadd[i_bp],
lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp],
gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp],
m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
if len(valid_obs[0])==0:
continue
n_bp = len(valid_obs[0])
dflux_bp = dflux[:,valid_obs[0]]
flux0_arr = flux_q[i_bp]
flux_tot = flux0_arr[:,None] + dflux_bp
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot,
snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp]
noise_single = flux_tot/snr_single_val
noise = np.sqrt(noise_coadd[:,None]**2+noise_single**2)
dflux_thresh = 5.0*noise
dflux_bp = np.abs(dflux_bp)
detected = (dflux_bp>=dflux_thresh)
snr_arr[:,valid_obs[0]] = dflux_bp/noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj,valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'],
photometry_mask_1d, obs_md_list,
filter_obs, proper_chip)
duration = (time.time()-t_before_chip)/3600.0
unq_out = -1*np.ones(n_obj, dtype=int)
mjd_out = -1.0*np.ones(n_obj, dtype=float)
snr_out = -1.0*np.ones(n_obj, dtype=float)
var_type_out = -1*np.ones(n_obj, dtype=int)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:]
if detected.any():
unq_out[i_obj] = chunk['simobjid'][i_obj]
first_dex = np.where(detected)[0].min()
mjd_out[i_obj] = mjd_obs[first_dex]
snr_out[i_obj] = snr_arr[i_obj, first_dex]
var_type_out[i_obj] = chunk['var_type'][i_obj]
valid = np.where(unq_out>=0)
unq_out = unq_out[valid]
mjd_out = mjd_out[valid]
var_type_out = var_type_out[valid]
snr_out = snr_out[valid]
with lock:
existing_keys = list(out_data.keys())
key_val = 0
if len(existing_keys)>0:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None, None)
out_data[key_val] = (unq_out, mjd_out,
snr_out, var_type_out)
def test_flare_magnitudes_mixed_with_dummy(self):
"""
Test that we get the expected magnitudes out
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
# load the quiescent SEDs of the objects in our catalog
sed_list = SedList(['lte028-5.0+0.5a+0.0.BT-Settl.spec.gz']*4,
[17.1, 17.2, 17.3, 17.4],
galacticAvList = [2.432, 1.876, 2.654, 2.364],
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
# calculate the quiescent fluxes of the objects in our catalog
baseline_fluxes = bp_dict.fluxListForSedList(sed_list)
bb_wavelen = np.arange(100.0, 1600.0, 0.1)
bb_flambda = blackbody_lambda(bb_wavelen*10.0, 9000.0)
# this data is taken from the setUpClass() classmethod above
t0_list = [456.2, 41006.2, 117.2, 10456.2]
av_list = [2.432, 1.876, 2.654, 2.364]
parallax_list = np.array([0.25, 0.15, 0.3, 0.22])
distance_list = 1.0/(206265.0*radiansFromArcsec(0.001*parallax_list))
distance_list *= 3.0857e18 # convert to cm
dtype = np.dtype([('id', int), ('u', float), ('g', float)])
photParams = PhotometricParameters()
ss = Sed()
quiet_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_dummy_quiet_cat.txt')
flare_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_dummy_flaring_cat.txt')
# loop over several MJDs and verify that, to within a
# milli-mag, our flaring model gives us the magnitudes
# expected, given the light curves specified in
# setUpClass()
for mjd in (59580.0, 60000.0, 70000.0, 80000.0):
obs = ObservationMetaData(mjd=mjd)
quiet_cat = QuiescentCatalog(db, obs_metadata=obs)
quiet_cat.write_catalog(quiet_cat_name)
flare_cat = FlaringCatalogDummy(db, obs_metadata=obs)
flare_cat.scratch_dir = self.scratch_dir
flare_cat._mlt_lc_file = self.mlt_lc_name
flare_cat.write_catalog(flare_cat_name)
quiescent_data = np.genfromtxt(quiet_cat_name, dtype=dtype, delimiter=',')
flaring_data = np.genfromtxt(flare_cat_name, dtype=dtype, delimiter=',')
self.assertGreater(len(quiescent_data), 2)
self.assertEqual(len(quiescent_data), len(flaring_data))
self.assertIn(3, flaring_data['id'])
for ix in range(len(flaring_data)):
obj_id = flaring_data['id'][ix]
self.assertEqual(obj_id, ix)
msg = ('failed on object %d; mjd %.2f\n u_quiet %e u_flare %e\n g_quiet %e g_flare %e' %
(obj_id, mjd, quiescent_data['u'][obj_id], flaring_data['u'][obj_id],
quiescent_data['g'][obj_id], flaring_data['g'][obj_id]))
self.assertEqual(quiescent_data['id'][obj_id], flaring_data['id'][obj_id], msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][0]),
quiescent_data['u'][obj_id], 3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][1]),
quiescent_data['g'][obj_id], 3, msg=msg)
if obj_id != 3:
# the models below are as specified in the
# setUpClass() method
if obj_id == 0 or obj_id == 1:
amp = 1.0e42
dt = 3652.5
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/100.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/100.0), 2))
elif obj_id==2:
amp = 2.0e41
dt = 365.25
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/50.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/50.0), 2))
# calculate the multiplicative effect of dust on a 9000K
# black body
bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda)
u_bb_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_flux = bb_sed.calcFlux(bp_dict['g'])
a_x, b_x = bb_sed.setupCCM_ab()
bb_sed.addDust(a_x, b_x, A_v=av_list[obj_id])
u_bb_dusty_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_dusty_flux = bb_sed.calcFlux(bp_dict['g'])
dust_u = u_bb_dusty_flux/u_bb_flux
dust_g = g_bb_dusty_flux/g_bb_flux
area = 4.0*np.pi*np.power(distance_list[obj_id], 2)
tot_u_flux = baseline_fluxes[obj_id][0] + u_flux*dust_u/area
tot_g_flux = baseline_fluxes[obj_id][1] + g_flux*dust_g/area
self.assertAlmostEqual(ss.magFromFlux(tot_u_flux), flaring_data['u'][obj_id],
3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(tot_g_flux), flaring_data['g'][obj_id],
3, msg=msg)
self.assertGreater(np.abs(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id]),
0.001, msg=msg)
self.assertGreater(np.abs(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id]),
0.001, msg=msg)
else:
self.assertAlmostEqual(flaring_data['g'][obj_id],
quiescent_data['g'][obj_id]+3*(mjd-59580.0)/10000.0,
3, msg=msg)
self.assertAlmostEqual(flaring_data['u'][obj_id],
quiescent_data['u'][obj_id]+2*(mjd-59580.0)/10000.0,
3, msg=msg)
if os.path.exists(quiet_cat_name):
os.unlink(quiet_cat_name)
if os.path.exists(flare_cat_name):
os.unlink(flare_cat_name)
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
def test_flare_magnitudes_mixed_with_none(self):
"""
Test that we get the expected magnitudes out
"""
db = MLT_test_DB(database=self.db_name, driver='sqlite')
# load the quiescent SEDs of the objects in our catalog
sed_list = SedList(['lte028-5.0+0.5a+0.0.BT-Settl.spec.gz']*4,
[17.1, 17.2, 17.3, 17.4],
galacticAvList = [2.432, 1.876, 2.654, 2.364],
fileDir=getPackageDir('sims_sed_library'),
specMap=defaultSpecMap)
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
# calculate the quiescent fluxes of the objects in our catalog
baseline_fluxes = bp_dict.fluxListForSedList(sed_list)
bb_wavelen = np.arange(100.0, 1600.0, 0.1)
bb = models.BlackBody(temperature=9000.0 * u.K, scale=1.0 * u.erg / (u.cm ** 2 * u.AA * u.s * u.sr))
bb_flambda = bb(bb_wavelen * u.nm).to_value()
# this data is taken from the setUpClass() classmethod above
t0_list = [456.2, 41006.2, 117.2, 10456.2]
av_list = [2.432, 1.876, 2.654, 2.364]
parallax_list = np.array([0.25, 0.15, 0.3, 0.22])
distance_list = 1.0/(206265.0*radiansFromArcsec(0.001*parallax_list))
distance_list *= 3.0857e18 # convert to cm
dtype = np.dtype([('id', int), ('u', float), ('g', float)])
photParams = PhotometricParameters()
ss = Sed()
quiet_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_none_quiet_cat.txt')
flare_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_none_flaring_cat.txt')
# loop over several MJDs and verify that, to within a
# milli-mag, our flaring model gives us the magnitudes
# expected, given the light curves specified in
# setUpClass()
for mjd in (59580.0, 60000.0, 70000.0, 80000.0):
obs = ObservationMetaData(mjd=mjd)
quiet_cat = QuiescentCatalog(db, obs_metadata=obs)
quiet_cat.write_catalog(quiet_cat_name)
flare_cat = FlaringCatalog(db, obs_metadata=obs)
flare_cat._mlt_lc_file = self.mlt_lc_name
flare_cat.write_catalog(flare_cat_name)
quiescent_data = np.genfromtxt(quiet_cat_name, dtype=dtype, delimiter=',')
flaring_data = np.genfromtxt(flare_cat_name, dtype=dtype, delimiter=',')
self.assertGreater(len(flaring_data), 3)
for ix in range(len(flaring_data)):
obj_id = flaring_data['id'][ix]
self.assertEqual(obj_id, ix)
# the models below are as specified in the
# setUpClass() method
if obj_id == 0 or obj_id == 1:
amp = 1.0e42
dt = 3652.5
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/100.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/100.0), 2))
elif obj_id==2:
amp = 2.0e41
dt = 365.25
t_min = flare_cat._survey_start - t0_list[obj_id]
tt = mjd - t_min
while tt > dt:
tt -= dt
u_flux = amp*(1.0+np.power(np.sin(tt/50.0), 2))
g_flux = amp*(1.0+np.power(np.cos(tt/50.0), 2))
else:
u_flux = 0.0
g_flux = 0.0
# calculate the multiplicative effect of dust on a 9000K
# black body
bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda)
u_bb_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_flux = bb_sed.calcFlux(bp_dict['g'])
a_x, b_x = bb_sed.setupCCM_ab()
bb_sed.addDust(a_x, b_x, A_v=av_list[obj_id])
u_bb_dusty_flux = bb_sed.calcFlux(bp_dict['u'])
g_bb_dusty_flux = bb_sed.calcFlux(bp_dict['g'])
dust_u = u_bb_dusty_flux/u_bb_flux
dust_g = g_bb_dusty_flux/g_bb_flux
area = 4.0*np.pi*np.power(distance_list[obj_id], 2)
tot_u_flux = baseline_fluxes[obj_id][0] + u_flux*dust_u/area
tot_g_flux = baseline_fluxes[obj_id][1] + g_flux*dust_g/area
msg = ('failed on object %d; mjd %.2f\n u_quiet %e u_flare %e\n g_quiet %e g_flare %e' %
(obj_id, mjd, quiescent_data['u'][obj_id], flaring_data['u'][obj_id],
quiescent_data['g'][obj_id], flaring_data['g'][obj_id]))
self.assertEqual(quiescent_data['id'][obj_id], flaring_data['id'][obj_id], msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][0]),
quiescent_data['u'][obj_id], 3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][1]),
quiescent_data['g'][obj_id], 3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(tot_u_flux), flaring_data['u'][obj_id],
3, msg=msg)
self.assertAlmostEqual(ss.magFromFlux(tot_g_flux), flaring_data['g'][obj_id],
3, msg=msg)
if obj_id != 3:
self.assertGreater(np.abs(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id]),
0.001, msg=msg)
self.assertGreater(np.abs(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id]),
0.001, msg=msg)
else:
self.assertEqual(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id], 0.0, msg=msg)
self.assertEqual(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id], 0.0, msg=msg)
if os.path.exists(quiet_cat_name):
os.unlink(quiet_cat_name)
if os.path.exists(flare_cat_name):
os.unlink(flare_cat_name)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5,
obs_md_list, proper_chip, variability_cache,
out_data):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
# from the overview paper
# table 2; take m5 row and add Delta m5 row
# to get down to airmass 1.2
m5_single = {}
m5_single['u'] = 23.57
m5_single['g'] = 24.65
m5_single['r'] = 24.21
m5_single['i'] = 23.79
m5_single['z'] = 23.21
m5_single['y'] = 22.31
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None] * n_t
dflux = np.zeros((n_obj, n_t), dtype=float)
dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_q = np.zeros((6, n_obj), dtype=float)
flux_coadd = np.zeros((6, n_obj), dtype=float)
mag_coadd = np.zeros((6, n_obj), dtype=float)
snr_coadd = np.zeros((6, n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1, exptime=30.0)
t_start_snr = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0 *
coadd_visits[bp])
flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp])
dflux_sub = dflux[:, valid_obs[0]]
assert dflux_sub.shape == (n_obj, len(valid_obs[0]))
dflux_mean = np.mean(dflux_sub, axis=1)
assert dflux_mean.shape == (n_obj, )
flux_coadd[i_bp] = flux_q[i_bp] + dflux_mean
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp], m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs == i_bp)
if len(valid_obs[0]) == 0:
continue
n_bp = len(valid_obs[0])
dflux_bp = dflux[:, valid_obs[0]]
flux0_arr = flux_q[i_bp]
flux_tot = flux0_arr[:, None] + dflux_bp
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot, snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp] / snr_coadd[i_bp]
noise_single = flux_tot / snr_single_val
noise = np.sqrt(noise_coadd[:, None]**2 + noise_single**2)
dflux_thresh = 5.0 * noise
dflux_bp = np.abs(dflux_bp)
detected = (dflux_bp >= dflux_thresh)
snr_arr[:, valid_obs[0]] = dflux_bp / noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj, valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'],
photometry_mask_1d, obs_md_list, filter_obs,
proper_chip)
duration = (time.time() - t_before_chip) / 3600.0
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj, :] & chip_mask[i_obj, :]
if detected.any():
unq = chunk['simobjid'][i_obj]
first_dex = np.where(detected)[0].min()
out_data[unq] = (mjd_obs[first_dex], snr_arr[i_obj, first_dex],
chunk['var_type'][i_obj])
if detected[0]:
ct_first += 1
else:
ct_at_all += 1
for lc_id in np.unique(lc_int_dex):
valid_stars = np.where(lc_int_dex == lc_id)
if len(valid_stars[0]) == 0:
continue
local_flux_factor = flux_factor[valid_stars]
local_dmag_max = np.zeros(len(valid_stars[0]), dtype=float)
local_mag_min = 41.0 * np.ones(len(valid_stars[0]), dtype=float)
for mag_name in mag_name_list:
mag_0 = chunk['%smag' % mag_name][valid_stars]
local_dust_factor = dust_factor[mag_name][valid_stars]
flux_0 = dummy_sed.fluxFromMag(mag_0)
dflux = dflux_dict[lc_id][
mag_name] * local_dust_factor * local_flux_factor
total_flux = flux_0 + dflux
mag = dummy_sed.magFromFlux(total_flux)
dmag = 2.5 * np.log10(1.0 + dflux / flux_0)
local_dmag_max = np.where(dmag > local_dmag_max, dmag,
local_dmag_max)
local_mag_min = np.where(mag < local_mag_min, mag,
local_mag_min)
dmag_max[valid_stars] = local_dmag_max
mag_min[valid_stars] = local_mag_min
ct_total += len(chunk)
with open(out_name, 'a') as out_file:
for i_star in range(len(chunk)):
if mag_min[i_star] > 24.89 or dmag_max[i_star] < 0.001:
ct_bad += 1
out_file.write(
uflare, gflare, rflare,
iflare, zflare, yflare,
tpeak) = light_curve_from_class(flare_type, 2.0, rng)
uflare *= flux_factor[i_star]
gflare *= flux_factor[i_star]
rflare *= flux_factor[i_star]
iflare *= flux_factor[i_star]
zflare *= flux_factor[i_star]
yflare *= flux_factor[i_star]
uflux = np.interp(tpeak, time, uflare)
gflux = np.interp(tpeak, time, gflare)
rflux = np.interp(tpeak, time, rflare)
iflux = np.interp(tpeak, time, iflare)
zflux = np.interp(tpeak, time, zflare)
yflux = np.interp(tpeak, time, yflare)
du += list(data['umag'][i_star] - ss.magFromFlux(base_u[i_star]+uflux))
dg += list(data['gmag'][i_star] - ss.magFromFlux(base_g[i_star]+gflux))
dr += list(data['rmag'][i_star] - ss.magFromFlux(base_r[i_star]+rflux))
di += list(data['imag'][i_star] - ss.magFromFlux(base_i[i_star]+iflux))
dz += list(data['zmag'][i_star] - ss.magFromFlux(base_z[i_star]+zflux))
dy += list(data['ymag'][i_star] - ss.magFromFlux(base_y[i_star]+yflux))
with open('delta_m_data.txt', 'w') as output_file:
for uu, gg, rr, ii, zz, yy in zip(du, dg, dr, di, dz, dy):
output_file.write('%e %e %e %e %e %e\n' % (uu, gg, rr, ii, zz, yy))
def _light_curves_from_query(self, cat_dict, query_result, grp, lc_per_field=None):
t_dict = {}
gamma_dict = {}
m5_dict = {}
t_min = None
t_max = None
for bp_name in cat_dict:
self.lsstBandpassDict[bp_name].sbTophi()
# generate a 2-D numpy array containing MJDs, m5, and photometric gamma values
# for each observation in the given bandpass
raw_array = np.array([[obs.mjd.TAI, obs.m5[bp_name],
calcGamma(self.lsstBandpassDict[bp_name],
obs.m5[obs.bandpass],
self.phot_params)]
for obs in grp if obs.bandpass == bp_name]).transpose()
if len(raw_array) > 0:
t_dict[bp_name] = raw_array[0]
m5_dict[bp_name] = raw_array[1]
gamma_dict[bp_name] = raw_array[2]
local_t_min = t_dict[bp_name].min()
local_t_max = t_dict[bp_name].max()
if t_min is None or local_t_min < t_min:
t_min = local_t_min
if t_max is None or local_t_max > t_max:
t_max = local_t_max
snobj = SNObject()
cat = cat_dict[list(cat_dict.keys())[0]] # does not need to be associated with a bandpass
dummy_sed = Sed()
n_actual_sn = 0 # how many SN have we actually delivered?
for chunk in query_result:
if lc_per_field is not None and n_actual_sn >= lc_per_field:
break
t_start_chunk = time.time()
for sn in cat.iter_catalog(query_cache=[chunk]):
sn_rng = self.sn_universe.getSN_rng(sn[1])
sn_t0 = self.sn_universe.drawFromT0Dist(sn_rng)
if sn[5] <= self.z_cutoff and np.isfinite(sn_t0) and \
sn_t0 < t_max + cat.maxTimeSNVisible and \
sn_t0 > t_min - cat.maxTimeSNVisible:
sn_c = self.sn_universe.drawFromcDist(sn_rng)
sn_x1 = self.sn_universe.drawFromx1Dist(sn_rng)
sn_x0 = self.sn_universe.drawFromX0Dist(sn_rng, sn_x1, sn_c, sn[4])
snobj.set(t0=sn_t0, c=sn_c, x1=sn_x1, x0=sn_x0, z=sn[5])
for bp_name in t_dict:
t_list = t_dict[bp_name]
m5_list = m5_dict[bp_name]
gamma_list = gamma_dict[bp_name]
bandpass = self.lsstBandpassDict[bp_name]
if len(t_list) == 0:
continue
if snobj.maxtime() >= t_list[0] and snobj.mintime() <= t_list[-1]:
active_dexes = np.where(np.logical_and(t_list >= snobj.mintime(),
t_list <= snobj.maxtime()))
t_active = t_list[active_dexes]
m5_active = m5_list[active_dexes]
gamma_active = gamma_list[active_dexes]
if len(t_active) > 0:
wave_ang = bandpass.wavelen*10.0
mask = np.logical_and(wave_ang > snobj.minwave(),
wave_ang < snobj.maxwave())
wave_ang = wave_ang[mask]
snobj.set(mwebv=sn[6])
sn_ff_buffer = snobj.flux(time=t_active, wave=wave_ang)*10.0
flambda_grid = np.zeros((len(t_active), len(bandpass.wavelen)))
for ff, ff_sn in zip(flambda_grid, sn_ff_buffer):
ff[mask] = np.where(ff_sn > 0.0, ff_sn, 0.0)
fnu_grid = flambda_grid*bandpass.wavelen* \
bandpass.wavelen*dummy_sed._physParams.nm2m* \
dummy_sed._physParams.ergsetc2jansky/dummy_sed._physParams.lightspeed
flux_list = \
(fnu_grid*bandpass.phi).sum(axis=1)*(bandpass.wavelen[1]-bandpass.wavelen[0])
acceptable = np.where(flux_list>0.0)
flux_error_list = flux_list[acceptable]/ \
calcSNR_m5(dummy_sed.magFromFlux(flux_list[acceptable]),
bandpass,
m5_active[acceptable], self.phot_params,
gamma=gamma_active[acceptable])
if len(acceptable) > 0:
n_actual_sn += 1
if lc_per_field is not None and n_actual_sn > lc_per_field:
break
if sn[0] not in self.truth_dict:
self.truth_dict[sn[0]] = {}
self.truth_dict[sn[0]]['t0'] = sn_t0
self.truth_dict[sn[0]]['x1'] = sn_x1
self.truth_dict[sn[0]]['x0'] = sn_x0
self.truth_dict[sn[0]]['c'] = sn_c
self.truth_dict[sn[0]]['z'] = sn[5]
self.truth_dict[sn[0]]['E(B-V)'] = sn[6]
if sn[0] not in self.mjd_dict:
self.mjd_dict[sn[0]] = {}
self.bright_dict[sn[0]] = {}
self.sig_dict[sn[0]] = {}
if bp_name not in self.mjd_dict[sn[0]]:
self.mjd_dict[sn[0]][bp_name] = []
self.bright_dict[sn[0]][bp_name] = []
self.sig_dict[sn[0]][bp_name] = []
for tt, ff, ee in zip(t_active[acceptable], flux_list[acceptable],
flux_error_list[0]):
self.mjd_dict[sn[0]][bp_name].append(tt)
self.bright_dict[sn[0]][bp_name].append(ff/3631.0)
self.sig_dict[sn[0]][bp_name].append(ee/3631.0)
print("chunk of ", len(chunk), " took ", time.time()-t_start_chunk)
def process_agn_chunk(chunk, filter_obs, mjd_obs, m5_obs,
coadd_m5, m5_single,
obs_md_list, proper_chip, out_data,
lock):
t_start_chunk = time.time()
#print('processing %d' % len(chunk))
ct_first = 0
ct_at_all = 0
ct_tot = 0
n_t = len(filter_obs)
n_obj = len(chunk)
agn_model = ExtraGalacticVariabilityModels()
dust_model = EBVbase()
with h5py.File('data/ebv_grid.h5', 'r') as in_file:
ebv_grid = in_file['ebv_grid'].value
extinction_grid = in_file['extinction_grid'].value
coadd_visits = {}
coadd_visits['u'] = 6
coadd_visits['g'] = 8
coadd_visits['r'] = 18
coadd_visits['i'] = 18
coadd_visits['z'] = 16
coadd_visits['y'] = 16
gamma_coadd = {}
for bp in 'ugrizy':
gamma_coadd[bp] = None
gamma_single = {}
for bp in 'ugrizy':
gamma_single[bp] = [None]*n_t
params = {}
params['agn_sfu'] = chunk['agn_sfu']
params['agn_sfg'] = chunk['agn_sfg']
params['agn_sfr'] = chunk['agn_sfr']
params['agn_sfi'] = chunk['agn_sfi']
params['agn_sfz'] = chunk['agn_sfz']
params['agn_sfy'] = chunk['agn_sfy']
params['agn_tau'] = chunk['agn_tau']
params['seed'] = chunk['id']+1
ebv = dust_model.calculateEbv(equatorialCoordinates=np.array([np.radians(chunk['ra']),
np.radians(chunk['dec'])]),
interp=True)
for i_bp, bp in enumerate('ugrizy'):
extinction_values = np.interp(ebv, ebv_grid, extinction_grid[i_bp])
chunk['%s_ab' % bp] += extinction_values
chunk['AGNLSST%s' % bp] += extinction_values
dmag = agn_model.applyAgn(np.where(np.array([True]*len(chunk))),
params, mjd_obs,
redshift=chunk['redshift'])
dmag_mean = np.mean(dmag, axis=2)
assert dmag_mean.shape == (6,n_obj)
dummy_sed = Sed()
lsst_bp = BandpassDict.loadTotalBandpassesFromFiles()
flux_gal = np.zeros((6,n_obj), dtype=float)
flux_agn_q = np.zeros((6,n_obj), dtype=float)
flux_coadd = np.zeros((6,n_obj), dtype=float)
mag_coadd = np.zeros((6,n_obj), dtype=float)
snr_coadd = np.zeros((6,n_obj), dtype=float)
snr_single = {}
snr_single_mag_grid = np.arange(14.0, 30.0, 0.05)
phot_params_single = PhotometricParameters(nexp=1,
exptime=30.0)
t_start_snr = time.time()
for i_bp, bp in enumerate('ugrizy'):
phot_params_coadd = PhotometricParameters(nexp=1,
exptime=30.0*coadd_visits[bp])
flux_gal[i_bp] = dummy_sed.fluxFromMag(chunk['%s_ab' % bp])
flux_agn_q[i_bp] = dummy_sed.fluxFromMag(chunk['AGNLSST%s' % bp] +
dmag_mean[i_bp,:])
flux_coadd[i_bp] = flux_gal[i_bp]+flux_agn_q[i_bp]
mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp])
(snr_coadd[i_bp],
gamma) = SNR.calcSNR_m5(mag_coadd[i_bp],
lsst_bp[bp],
coadd_m5[bp],
phot_params_coadd)
(snr_single[bp],
gamma) = SNR.calcSNR_m5(snr_single_mag_grid,
lsst_bp[bp],
m5_single[bp],
phot_params_single)
#print('got all snr in %e' % (time.time()-t_start_snr))
t_start_obj = time.time()
photometry_mask = np.zeros((n_obj, n_t), dtype=bool)
photometry_mask_1d = np.zeros(n_obj, dtype=bool)
snr_arr = np.zeros((n_obj, n_t), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
n_bp = len(valid_obs[0])
if n_bp == 0:
continue
mag0_arr = chunk['AGNLSST%s' % bp]
dmag_bp = dmag[i_bp][:,valid_obs[0]]
assert dmag_bp.shape == (n_obj, n_bp)
agn_flux_tot = dummy_sed.fluxFromMag(mag0_arr[:,None]+dmag_bp)
q_flux = flux_agn_q[i_bp]
agn_dflux = np.abs(agn_flux_tot-q_flux[:,None])
flux_tot = flux_gal[i_bp][:, None] + agn_flux_tot
assert flux_tot.shape == (n_obj, n_bp)
mag_tot = dummy_sed.magFromFlux(flux_tot)
snr_single_val = np.interp(mag_tot,
snr_single_mag_grid,
snr_single[bp])
noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp]
noise_single = flux_tot/snr_single_val
noise = np.sqrt(noise_coadd[:,None]**2 + noise_single**2)
dflux_thresh = 5.0*noise
detected = (agn_dflux>=dflux_thresh)
assert detected.shape == (n_obj, n_bp)
snr_arr[:,valid_obs[0]] = agn_dflux/noise
for i_obj in range(n_obj):
if detected[i_obj].any():
photometry_mask_1d[i_obj] = True
photometry_mask[i_obj, valid_obs[0]] = detected[i_obj]
t_before_chip = time.time()
chip_mask = apply_focal_plane(chunk['ra'], chunk['dec'],
photometry_mask_1d, obs_md_list,
filter_obs, proper_chip)
duration = (time.time()-t_before_chip)/3600.0
unq_out = -1*np.ones(n_obj, dtype=int)
mjd_out = -1.0*np.ones(n_obj, dtype=float)
snr_out = -1.0*np.ones(n_obj, dtype=float)
for i_obj in range(n_obj):
if photometry_mask_1d[i_obj]:
detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:]
if detected.any():
unq_out[i_obj] = chunk['galtileid'][i_obj]
first_dex = np.where(detected)[0].min()
mjd_out[i_obj] = mjd_obs[first_dex]
snr_out[i_obj] = snr_arr[i_obj, first_dex]
valid = np.where(unq_out>=0)
unq_out = unq_out[valid]
mjd_out = mjd_out[valid]
snr_out = snr_out[valid]
with lock:
existing_keys = list(out_data.keys())
if len(existing_keys) == 0:
key_val = 0
else:
key_val = max(existing_keys)
while key_val in existing_keys:
key_val += 1
out_data[key_val] = (None, None, None)
out_data[key_val] = (unq_out, mjd_out, snr_out)
