def test_one_by_one(self):
"""
test that running RA, Dec pairs in one at a time gives the same
results as running them in in batches
"""
ra = 145.0
dec = -25.0
obs = ObservationMetaData(pointingRA=ra, pointingDec=dec,
mjd=59580.0, rotSkyPos=113.0)
rng = np.random.RandomState(100)
theta = rng.random_sample(100)*2.0*np.pi
rr = rng.random_sample(len(theta))*2.0
ra_list = ra + rr*np.cos(theta)
dec_list = dec + rr*np.sin(theta)
name_control = chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
is_none = 0
for ra, dec, name in zip(ra_list, dec_list, name_control):
test_name = chipNameFromRaDecLSST(ra, dec, obs_metadata=obs)
self.assertEqual(test_name, name)
if test_name is None:
is_none += 1
self.assertGreater(is_none, 0)
self.assertLess(is_none, (3*len(ra_list))//4)
def test_one_by_one(self):
"""
test that running RA, Dec pairs in one at a time gives the same
results as running them in in batches
"""
ra = 145.0
dec = -25.0
obs = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
mjd=59580.0,
rotSkyPos=113.0)
rng = np.random.RandomState(100)
theta = rng.random_sample(100) * 2.0 * np.pi
rr = rng.random_sample(len(theta)) * 2.0
ra_list = ra + rr * np.cos(theta)
dec_list = dec + rr * np.sin(theta)
name_control = chipNameFromRaDecLSST(ra_list,
dec_list,
obs_metadata=obs)
is_none = 0
for ra, dec, name in zip(ra_list, dec_list, name_control):
test_name = chipNameFromRaDecLSST(ra, dec, obs_metadata=obs)
self.assertEqual(test_name, name)
if test_name is None:
is_none += 1
self.assertGreater(is_none, 0)
self.assertLess(is_none, (3 * len(ra_list)) // 4)
def test_chip_name_from_ra_dec_degrees(self):
"""
test that chipNameFromRaDecLSST agrees with chipNameFromRaDec
"""
n_obj = 1000
raP = 112.1
decP = -34.1
obs = ObservationMetaData(pointingRA=raP, pointingDec=decP,
rotSkyPos=45.0, mjd=43000.0)
rng = np.random.RandomState(8731)
rr = rng.random_sample(n_obj)*1.75
theta = rng.random_sample(n_obj)*2.0*np.pi
ra_list = raP + rr*np.cos(theta)
dec_list = decP + rr*np.sin(theta)
control_name_list = chipNameFromRaDec(ra_list, dec_list,
obs_metadata=obs,
camera=self.camera)
test_name_list = chipNameFromRaDecLSST(ra_list, dec_list,
obs_metadata=obs)
try:
np.testing.assert_array_equal(control_name_list.astype(str),
test_name_list.astype(str))
except AssertionError:
n_problematic = 0
for ii, (c_n, t_n) in enumerate(zip(control_name_list.astype(str), test_name_list.astype(str))):
if c_n != t_n:
x_pix, y_pix = pixelCoordsFromRaDecLSST(ra_list[ii], dec_list[ii], obs_metadata=obs)
if c_n != 'None':
n_problematic += 1
if n_problematic > 0:
raise
self.assertLessEqual(len(np.where(np.char.rfind(test_name_list.astype(str), 'None') >= 0)[0]),
n_obj/10)
# test that exceptions are raised when incomplete ObservationMetaData are used
obs = ObservationMetaData(pointingRA=raP, pointingDec=decP, mjd=59580.0)
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
self.assertIn("rotSkyPos", context.exception.args[0])
obs = ObservationMetaData(pointingRA=raP, pointingDec=decP, rotSkyPos=35.0)
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
self.assertIn("mjd", context.exception.args[0])
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list)
self.assertIn("ObservationMetaData", context.exception.args[0])
# check that exceptions are raised when ra_list, dec_list are of the wrong shape
obs = ObservationMetaData(pointingRA=raP, pointingDec=decP, rotSkyPos=24.0, mjd=43000.0)
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list[:5], obs_metadata=obs)
self.assertIn("chipNameFromRaDecLSST", context.exception.args[0])
def test_outside_radius(self):
"""
Test that methods can gracefully handle points
outside of the focal plane
"""
rng = np.random.RandomState(7123)
ra = 145.0
dec = -25.0
obs = ObservationMetaData(pointingRA=ra, pointingDec=dec,
mjd=59580.0, rotSkyPos=113.0)
rr = rng.random_sample(100)*5.0
self.assertGreater(rr.max(), 4.5)
theta = rng.random_sample(100)*2.0*np.pi
ra_vec = ra + rr*np.cos(theta)
dec_vec = dec + rr*np.sin(theta)
chip_name_list = chipNameFromRaDecLSST(ra_vec, dec_vec,
obs_metadata=obs,
band='u')
rr = angularSeparation(ra, dec, ra_vec, dec_vec)
ct_none = 0
for rr, name in zip(rr, chip_name_list):
if rr > 2.0:
self.assertIsNone(name)
ct_none += 1
self.assertGreater(ct_none, 0)
def test_outside_radius(self):
"""
Test that methods can gracefully handle points
outside of the focal plane
"""
rng = np.random.RandomState(7123)
ra = 145.0
dec = -25.0
obs = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
mjd=59580.0,
rotSkyPos=113.0)
rr = rng.random_sample(100) * 5.0
self.assertGreater(rr.max(), 4.5)
theta = rng.random_sample(100) * 2.0 * np.pi
ra_vec = ra + rr * np.cos(theta)
dec_vec = dec + rr * np.sin(theta)
chip_name_list = chipNameFromRaDecLSST(ra_vec,
dec_vec,
obs_metadata=obs,
band='u')
rr = angularSeparation(ra, dec, ra_vec, dec_vec)
ct_none = 0
for rr, name in zip(rr, chip_name_list):
if rr > 2.0:
self.assertIsNone(name)
ct_none += 1
self.assertGreater(ct_none, 0)
def ssoInCameraFov(self, ephems, obsData):
"""Determine which observations are within the actual camera footprint for a series of observations.
Note that ephems and obsData must be the same length.
Parameters
----------
ephems : np.recarray
Ephemerides for the objects.
obsData : np.recarray
Observation pointings.
Returns
-------
np.ndarray
Returns the indexes of the numpy array of the object observations which are inside the fov.
"""
if not hasattr(self, 'camera'):
self._setupCamera()
epoch = 2000.0
# See if the object is within 'rFov' of the center of the boresight.
idxObsRough = self._ssoInCircleFov(ephems, obsData, rFov=2.1)
# Then test for the camera footprint exactly.
idxObs = []
for idx in idxObsRough:
mjd_date = obsData[idx][self.obsTimeCol]
if self.obsTimeScale == 'TAI':
mjd = ModifiedJulianDate(TAI=mjd_date)
elif self.obsTimeScale == 'UTC':
mjd = ModifiedJulianDate(UTC=mjd_date)
else:
warnings.warn(
'Expected timescale of TAI or UTC, but did not match. Using TAI.'
)
mjd = ModifiedJulianDate(TAI=mjd_date)
if not self.obsDegrees:
obs_metadata = ObservationMetaData(
pointingRA=np.degrees(obsData[idx][self.obsRA]),
pointingDec=np.degrees(obsData[idx][self.obsDec]),
rotSkyPos=np.degrees(obsData[idx][self.obsRotSkyPos]),
mjd=mjd)
else:
obs_metadata = ObservationMetaData(
pointingRA=obsData[idx][self.obsRA],
pointingDec=obsData[idx][self.obsDec],
rotSkyPos=obsData[idx][self.obsRotSkyPos],
mjd=mjd)
# Catch the warnings from astropy about the time being in the future.
with warnings.catch_warnings(record=False):
warnings.simplefilter('ignore')
chipName = chipNameFromRaDecLSST(ra=ephems['ra'][idx],
dec=ephems['dec'][idx],
epoch=epoch,
obs_metadata=obs_metadata)
if chipName != None:
tt = self.ccd_type_dict[self.camera[chipName].getType()]
if tt == 'science':
idxObs.append(idx)
idxObs = np.array(idxObs, int)
return idxObs
def test_chip_name(self):
"""
Test that chipNameFromRaDecLSST with non-zero proper motion etc.
agrees with chipNameFromPupilCoords when pupilCoords are
calculated with the same proper motion, etc.
"""
(obs, ra_list, dec_list, pm_ra_list, pm_dec_list, parallax_list,
v_rad_list) = self.set_data(11)
for is_none in ('pm_ra', 'pm_dec', 'parallax', 'v_rad'):
pm_ra = pm_ra_list
pm_dec = pm_dec_list
parallax = parallax_list
v_rad = v_rad_list
if is_none == 'pm_ra':
pm_ra = None
elif is_none == 'pm_dec':
pm_dec = None
elif is_none == 'parallax':
parallax = None
elif is_none == 'v_rad':
v_rad = None
xp, yp = pupilCoordsFromRaDec(ra_list,
dec_list,
pm_ra=pm_ra,
pm_dec=pm_dec,
parallax=parallax,
v_rad=v_rad,
obs_metadata=obs)
name_control = chipNameFromPupilCoords(xp, yp, camera=self.camera)
name_test = chipNameFromRaDecLSST(ra_list,
dec_list,
pm_ra=pm_ra,
pm_dec=pm_dec,
parallax=parallax,
v_rad=v_rad,
obs_metadata=obs)
name_radians = _chipNameFromRaDecLSST(
np.radians(ra_list),
np.radians(dec_list),
pm_ra=radiansFromArcsec(pm_ra),
pm_dec=radiansFromArcsec(pm_dec),
parallax=radiansFromArcsec(parallax),
v_rad=v_rad,
obs_metadata=obs)
np.testing.assert_array_equal(name_control, name_test)
np.testing.assert_array_equal(name_control, name_radians)
self.assertGreater(len(np.unique(name_control.astype(str))), 4)
self.assertLess(len(np.where(np.equal(name_control, None))[0]),
len(name_control) / 4)
def test_chip_center(self):
"""
Test that, if we ask for the chip at the bore site,
we get back 'R:2,2 S:1,1'
"""
ra = 145.0
dec = -25.0
obs = ObservationMetaData(pointingRA=ra, pointingDec=dec,
mjd=59580.0, rotSkyPos=113.0)
name = chipNameFromRaDecLSST(ra, dec, obs_metadata=obs)
self.assertEqual(name, 'R:2,2 S:1,1')
def inCameraFov(self,
ephems,
obsData,
epoch=2000.0,
timeCol='observationStartMJD'):
"""Determine which observations are within the actual camera footprint for a series of observations.
Parameters
----------
ephems : np.recarray
Ephemerides for the objects, with RA and Dec as 'ra' and 'dec' columns (in degrees).
obsData : np.recarray
Observation pointings, with RA and Dec as 'ra' and 'dec' columns (in degrees).
The telescope rotation angle should be in 'rotSkyPos' (in degrees), and the time of each
pointing should be in the 'expMJD' column.
epoch: float, opt
The epoch of the ephemerides and pointing data. Default 2000.0.
Returns
-------
np.ndarray
Returns the indexes of the numpy array of the object observations which are inside the fov.
"""
# See if the object is within 'rFov' of the center of the boresight.
sep = angularSeparation(ephems['ra'], ephems['dec'], obsData['ra'],
obsData['dec'])
idxObsRough = np.where(sep <= 2.1)[0]
# Or go on and use the camera footprint.
idxObs = []
for idx in idxObsRough:
mjd_date = obsData[idx][timeCol]
mjd = ModifiedJulianDate(TAI=mjd_date)
obs_metadata = ObservationMetaData(
pointingRA=obsData[idx]['ra'],
pointingDec=obsData[idx]['dec'],
rotSkyPos=obsData[idx]['rotSkyPos'],
mjd=mjd)
# Catch the warnings from astropy about the time being in the future.
with warnings.catch_warnings(record=False):
warnings.simplefilter('ignore')
chipName = chipNameFromRaDecLSST(ra=ephems['ra'][idx],
dec=ephems['dec'][idx],
epoch=epoch,
obs_metadata=obs_metadata)
if chipName != None:
tt = self.ccd_type_dict[self.camera[chipName].getType()]
if tt == 'science':
idxObs.append(idx)
idxObs = np.array(idxObs, int)
return idxObs
def _actually_on_chip(ra, dec, obs_md):
"""
Take a numpy array of RA in degrees, a numpy array of Decin degrees
and an ObservationMetaData and return a boolean array indicating
which of the objects are actually on a chip and which are not
"""
out_arr = np.array([False]*len(ra))
d_ang = 2.11
good_radii = np.where(angularSeparation(ra, dec, obs_md.pointingRA, obs_md.pointingDec)<d_ang)
if len(good_radii[0]) > 0:
chip_names = chipNameFromRaDecLSST(ra[good_radii], dec[good_radii], obs_metadata=obs_md).astype(str)
vals = np.where(np.char.find(chip_names, 'None')==0, False, True)
out_arr[good_radii] = vals
return out_arr
def test_chip_center(self):
"""
Test that, if we ask for the chip at the bore site,
we get back 'R:2,2 S:1,1'
"""
ra = 145.0
dec = -25.0
obs = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
mjd=59580.0,
rotSkyPos=113.0)
name = chipNameFromRaDecLSST(ra, dec, obs_metadata=obs)
self.assertEqual(name, 'R:2,2 S:1,1')
def test_chip_name(self):
"""
Test that chipNameFromRaDecLSST with non-zero proper motion etc.
agrees with chipNameFromPupilCoords when pupilCoords are
calculated with the same proper motion, etc.
"""
(obs, ra_list, dec_list,
pm_ra_list, pm_dec_list,
parallax_list, v_rad_list) = self.set_data(11)
for is_none in ('pm_ra', 'pm_dec', 'parallax', 'v_rad'):
pm_ra = pm_ra_list
pm_dec = pm_dec_list
parallax = parallax_list
v_rad = v_rad_list
if is_none == 'pm_ra':
pm_ra = None
elif is_none == 'pm_dec':
pm_dec = None
elif is_none == 'parallax':
parallax = None
elif is_none == 'v_rad':
v_rad = None
xp, yp = pupilCoordsFromRaDec(ra_list, dec_list,
pm_ra=pm_ra, pm_dec=pm_dec,
parallax=parallax, v_rad=v_rad,
obs_metadata=obs)
name_control = chipNameFromPupilCoords(xp, yp, camera=self.camera)
name_test = chipNameFromRaDecLSST(ra_list, dec_list,
pm_ra=pm_ra, pm_dec=pm_dec,
parallax=parallax, v_rad=v_rad,
obs_metadata=obs)
name_radians = _chipNameFromRaDecLSST(np.radians(ra_list), np.radians(dec_list),
pm_ra=radiansFromArcsec(pm_ra),
pm_dec=radiansFromArcsec(pm_dec),
parallax=radiansFromArcsec(parallax), v_rad=v_rad,
obs_metadata=obs)
np.testing.assert_array_equal(name_control, name_test)
np.testing.assert_array_equal(name_control, name_radians)
self.assertGreater(len(np.unique(name_control.astype(str))), 4)
self.assertLess(len(np.where(np.equal(name_control, None))[0]), len(name_control)/4)
def test_chipName(self):
"""
Simply verify that CatSim puts the sources on the right chip
"""
self.assertGreater(len(self.data), 10)
for ix in range(len(self.data)):
obs = ObservationMetaData(pointingRA=self.data['pointingRA'][ix],
pointingDec=self.data['pointingDec'][ix],
rotSkyPos=self.data['rotSkyPos'][ix],
mjd=59580.0)
chipName = chipNameFromRaDecLSST(self.data['objRA'][ix],
self.data['objDec'][ix],
obs_metadata=obs)
self.assertEqual(chipName.replace(',','').replace(':','').replace(' ',''),
self.data['chipName'][ix])
def ssoInFov(self,
interpfuncs,
simdata,
rFov=1.75,
useCamera=True,
simdataRaCol='fieldRA',
simdataDecCol='fieldDec',
simdataExpMJDCol='expMJD'):
"""
Return the indexes of the simdata observations where the object was inside the fov.
"""
# See if the object is within 'rFov' of the center of the boresight.
raSso = interpfuncs['ra'](simdata[simdataExpMJDCol])
decSso = interpfuncs['dec'](simdata[simdataExpMJDCol])
sep = angularSeparation(raSso, decSso,
np.degrees(simdata[simdataRaCol]),
np.degrees(simdata[simdataDecCol]))
if not useCamera:
idxObsRough = np.where(sep < rFov)[0]
return idxObsRough
# Or go on and use the camera footprint.
idxObs = []
idxObsRough = np.where(sep < self.cameraFov)[0]
for idx in idxObsRough:
mjd_date = simdata[idx][simdataExpMJDCol]
mjd = ModifiedJulianDate(TAI=mjd_date)
obs_metadata = ObservationMetaData(
pointingRA=np.degrees(simdata[idx][simdataRaCol]),
pointingDec=np.degrees(simdata[idx][simdataDecCol]),
rotSkyPos=np.degrees(simdata[idx]['rotSkyPos']),
mjd=mjd)
raObj = np.array(
[interpfuncs['ra'](simdata[idx][simdataExpMJDCol])])
decObj = np.array(
[interpfuncs['dec'](simdata[idx][simdataExpMJDCol])])
# Catch the warnings from astropy about the time being in the future.
with warnings.catch_warnings(record=False):
warnings.simplefilter('ignore')
chipNames = chipNameFromRaDecLSST(ra=raObj,
dec=decObj,
epoch=self.epoch,
obs_metadata=obs_metadata)
if chipNames != [None]:
idxObs.append(idx)
idxObs = np.array(idxObs)
return idxObs
def test_chipName(self):
"""
Simply verify that CatSim puts the sources on the right chip
"""
self.assertGreater(len(self.data), 10)
for ix in range(len(self.data)):
obs = ObservationMetaData(pointingRA=self.data['pointingRA'][ix],
pointingDec=self.data['pointingDec'][ix],
rotSkyPos=self.data['rotSkyPos'][ix],
mjd=59580.0)
chipName = chipNameFromRaDecLSST(self.data['objRA'][ix],
self.data['objDec'][ix],
obs_metadata=obs)
self.assertEqual(
chipName.replace(',', '').replace(':', '').replace(' ', ''),
self.data['chipName'][ix])
def apply_focal_plane(ra, dec, photometry_mask_1d,
obs_md_list, filter_obs, proper_chip):
fov_radius = 1.75
n_obj = len(ra)
n_t = len(filter_obs)
if not proper_chip:
seed = int(np.round(ra[0]*1000.0+
dec[0]*1000.0+
obs_md_list[0].mjd.TAI))
rng = np.random.RandomState(seed)
chip_mask = np.zeros((n_obj, n_t), dtype=bool)
for i_t, (obs, i_bp) in enumerate(zip(obs_md_list, filter_obs)):
if proper_chip:
chip_name= chipNameFromRaDecLSST(ra[photometry_mask_1d],
dec[photometry_mask_1d],
obs_metadata=obs,
band='ugrizy'[i_bp])
valid_chip = (np.char.find(chip_name.astype(str), 'None') == -1)
else:
ra_valid = ra[photometry_mask_1d]
dec_valid = dec[photometry_mask_1d]
dd = angularSeparation(ra_valid,
dec_valid,
obs.pointingRA, obs.pointingDec)
#focal_plane_roll = rng.random_sample(len(ra_valid))
#valid_chip = (dd<=fov_radius)&(focal_plane_roll<0.9)
valid_chip = (dd<=fov_radius)
chip_mask[photometry_mask_1d, i_t] = valid_chip
return chip_mask
def test_chip_name_from_ra_dec_degrees(self):
"""
test that chipNameFromRaDecLSST agrees with chipNameFromRaDec
"""
n_obj = 1000
raP = 112.1
decP = -34.1
obs = ObservationMetaData(pointingRA=raP,
pointingDec=decP,
rotSkyPos=45.0,
mjd=43000.0)
rng = np.random.RandomState(8731)
rr = rng.random_sample(n_obj) * 1.75
theta = rng.random_sample(n_obj) * 2.0 * np.pi
ra_list = raP + rr * np.cos(theta)
dec_list = decP + rr * np.sin(theta)
control_name_list = chipNameFromRaDec(ra_list,
dec_list,
obs_metadata=obs,
camera=self.camera)
test_name_list = chipNameFromRaDecLSST(ra_list,
dec_list,
obs_metadata=obs)
np.testing.assert_array_equal(control_name_list.astype(str),
test_name_list.astype(str))
self.assertLessEqual(
len(
np.where(
np.char.rfind(test_name_list.astype(str), 'None') >= 0)
[0]), n_obj / 10)
# test that exceptions are raised when incomplete ObservationMetaData are used
obs = ObservationMetaData(pointingRA=raP,
pointingDec=decP,
mjd=59580.0)
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
self.assertIn("rotSkyPos", context.exception.args[0])
obs = ObservationMetaData(pointingRA=raP,
pointingDec=decP,
rotSkyPos=35.0)
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
self.assertIn("mjd", context.exception.args[0])
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list)
self.assertIn("ObservationMetaData", context.exception.args[0])
# check that exceptions are raised when ra_list, dec_list are of the wrong shape
obs = ObservationMetaData(pointingRA=raP,
pointingDec=decP,
rotSkyPos=24.0,
mjd=43000.0)
with self.assertRaises(RuntimeError) as context:
chipNameFromRaDecLSST(ra_list, dec_list[:5], obs_metadata=obs)
self.assertIn("chipNameFromRaDecLSST", context.exception.args[0])
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z': 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str,
_max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self,
valid_dexes,
params,
expmjd,
variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros(
(6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star] * np.cos(
params['per'][i_star] * expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin,
AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST,
AstrometryStars, Variability,
InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
@compound('delta_umag', 'delta_gmag', 'delta_rmag', 'delta_imag',
'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(
self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' %
self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name,
driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[
obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid) / np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) +
obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid,
star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float),
('band', int), ('q_flux', float),
('dflux', float), ('q_snr', float),
('tot_snr', float), ('ra', float),
('dec', float), ('chipNum', int),
('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float),
('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query,
dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add(
(alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq // 1024) - 1
self.assertAlmostEqual(self.pmra_truth[obj_dex],
0.001 * alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex],
0.001 * alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex],
0.001 * alert_data['parallax'][i_obj],
4)
ra_truth, dec_truth = applyProperMotion(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
distance_arcsec = 3600.0 * distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (
ra_truth, dec_truth, alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDecLSST(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
chipnum = int(
chipname.replace('R', '').replace('S', '').replace(
' ', '').replace(';', '').replace(',',
'').replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDecLSST(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5 * np.log10(1.0 + alert_data['dflux'][i_obj] /
alert_data['q_flux'][i_obj])
self.assertAlmostEqual(
true_lc_dict[alert_data['uniqueId'][i_obj]][
alert_data['obshistId'][i_obj]], dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z',
'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(
self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(
self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr / alert_data['q_snr'][i_obj], 1.0,
4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name], m5,
phot_params)
self.assertAlmostEqual(snr / alert_data['tot_snr'][i_obj], 1.0,
4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set),
len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int), ('ra', float),
('dec', float), ('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(
astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj] // 1024) - 1
ra_truth, dec_truth = applyProperMotion(
self.ra_truth[obj_dex],
self.dec_truth[obj_dex],
self.pmra_truth[obj_dex],
self.pmdec_truth[obj_dex],
self.px_truth[obj_dex],
self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0 * distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set),
n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
rot_tel = getRotTelPos(ra_pointing, dec_pointing, obs_pointing, rot_sky)
file_handle.write('rottelpos %.6f\n' % rot_tel)
file_handle.write('rotskypos %.6f\n' % rot_sky)
file_handle.write('obshistid %d\n' % obshistid)
from lsst.sims.coordUtils import chipNameFromRaDecLSST
ra_list = np.array([ra_pointing, ra_pointing, ra_pointing+0.5])
dec_list = np.array([dec_pointing, dec_pointing - 0.5, dec_pointing])
sed_name = 'starSED/kurucz/km30_5000.fits_g10_5040.gz'
with open('phosim_rot_control.txt', 'w') as control_file:
for ix, rot_sky in enumerate(np.arange(0.0, 271.0, 30.0)):
obs = ObservationMetaData(pointingRA=ra_pointing,
pointingDec=dec_pointing,
mjd=obs_pointing.mjd.TAI,
rotSkyPos=rot_sky)
file_name = 'coordutils_catalogs/phosim_rot_catalog_%.1f.txt' % rot_sky
with open(file_name, 'w') as output_file:
write_header(output_file, rot_sky, ix+1)
for i_obj in range(len(ra_list)):
output_file.write('object %d %e %e 22.0 %s 0 0 0 0 0 0 point none none\n' %
(i_obj+1, ra_list[i_obj], dec_list[i_obj], sed_name))
chip_names = chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
for i_obj in range(len(ra_list)):
control_file.write('%s %d %s\n' % (file_name, i_obj+1, chip_names[i_obj]))
control_file.write('\n')
def test_alert_data_generation(self):
dmag_cutoff = 0.005
mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5}
_max_var_param_str = self.max_str_len
class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject):
objid = 'star_alert'
tableid = 'stars'
idColKey = 'simobjid'
raColName = 'ra'
decColName = 'dec'
objectTypeId = 0
columns = [('raJ2000', 'ra*0.01745329252'),
('decJ2000', 'dec*0.01745329252'),
('parallax', 'px*0.01745329252/3600.0'),
('properMotionRa', 'pmra*0.01745329252/3600.0'),
('properMotionDec', 'pmdec*0.01745329252/3600.0'),
('radialVelocity', 'vrad'),
('variabilityParameters', 'varParamStr', str, _max_var_param_str)]
class TestAlertsVarCatMixin(object):
@register_method('alert_test')
def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None):
if len(params) == 0:
return np.array([[], [], [], [], [], []])
if isinstance(expmjd, numbers.Number):
dmags_out = np.zeros((6, self.num_variable_obj(params)))
else:
dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd)))
for i_star in range(self.num_variable_obj(params)):
if params['amp'][i_star] is not None:
dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd)
for i_filter in range(6):
dmags_out[i_filter][i_star] = dmags
return dmags_out
class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog):
pass
class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars,
Variability, InstanceCatalog):
column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert']
@compound('delta_umag', 'delta_gmag', 'delta_rmag',
'delta_imag', 'delta_zmag', 'delta_ymag')
def get_TruthVariability(self):
return self.applyVariability(self.column_by_name('varParamStr'))
@cached
def get_dmagAlert(self):
return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass)
@cached
def get_magAlert(self):
return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \
self.column_by_name('dmagAlert')
star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite')
# assemble the true light curves for each object; we need to figure out
# if their np.max(dMag) ever goes over dmag_cutoff; then we will know if
# we are supposed to simulate them
true_lc_dict = {}
true_lc_obshistid_dict = {}
is_visible_dict = {}
obs_dict = {}
max_obshistid = -1
n_total_observations = 0
for obs in self.obs_list:
obs_dict[obs.OpsimMetaData['obsHistID']] = obs
obshistid = obs.OpsimMetaData['obsHistID']
if obshistid > max_obshistid:
max_obshistid = obshistid
cat = TestAlertsTruthCat(star_db, obs_metadata=obs)
for line in cat.iter_catalog():
if line[1] is None:
continue
n_total_observations += 1
if line[0] not in true_lc_dict:
true_lc_dict[line[0]] = {}
true_lc_obshistid_dict[line[0]] = []
true_lc_dict[line[0]][obshistid] = line[2]
true_lc_obshistid_dict[line[0]].append(obshistid)
if line[0] not in is_visible_dict:
is_visible_dict[line[0]] = False
if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]:
is_visible_dict[line[0]] = True
obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2)))
skipped_due_to_mag = 0
objects_to_simulate = []
obshistid_unqid_set = set()
for obj_id in true_lc_dict:
dmag_max = -1.0
for obshistid in true_lc_dict[obj_id]:
if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max:
dmag_max = np.abs(true_lc_dict[obj_id][obshistid])
if dmag_max >= dmag_cutoff:
if not is_visible_dict[obj_id]:
skipped_due_to_mag += 1
continue
objects_to_simulate.append(obj_id)
for obshistid in true_lc_obshistid_dict[obj_id]:
obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid)
self.assertGreater(len(objects_to_simulate), 10)
self.assertGreater(skipped_due_to_mag, 0)
log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt')
alert_gen = AlertDataGenerator(testing=True)
alert_gen.subdivide_obs(self.obs_list, htmid_level=6)
for htmid in alert_gen.htmid_list:
alert_gen.alert_data_from_htmid(htmid, star_db,
photometry_class=TestAlertsVarCat,
output_prefix='alert_test',
output_dir=self.output_dir,
dmag_cutoff=dmag_cutoff,
log_file_name=log_file_name)
dummy_sed = Sed()
bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
phot_params = PhotometricParameters()
# First, verify that the contents of the sqlite files are all correct
n_tot_simulated = 0
alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, '
alert_query += 'meta.band, quiescent.flux, alert.dflux, '
alert_query += 'quiescent.snr, alert.snr, '
alert_query += 'alert.ra, alert.dec, alert.chipNum, '
alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, '
alert_query += 'ast.parallax '
alert_query += 'FROM alert_data AS alert '
alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId '
alert_query += 'INNER JOIN quiescent_flux AS quiescent '
alert_query += 'ON quiescent.uniqueId=alert.uniqueId '
alert_query += 'AND quiescent.band=meta.band '
alert_query += 'INNER JOIN baseline_astrometry AS ast '
alert_query += 'ON ast.uniqueId=alert.uniqueId'
alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int),
('TAI', float), ('band', int),
('q_flux', float), ('dflux', float),
('q_snr', float), ('tot_snr', float),
('ra', float), ('dec', float),
('chipNum', int), ('xPix', float), ('yPix', float),
('pmRA', float), ('pmDec', float), ('parallax', float)])
sqlite_file_list = os.listdir(self.output_dir)
n_tot_simulated = 0
obshistid_unqid_simulated_set = set()
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype)
if len(alert_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI'])
for i_obj in range(len(alert_data)):
n_tot_simulated += 1
obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) +
alert_data['obshistId'][i_obj])
unq = alert_data['uniqueId'][i_obj]
obj_dex = (unq//1024)-1
self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4)
self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4)
self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4)
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
alert_data['ra'][i_obj], alert_data['dec'][i_obj])
distance_arcsec = 3600.0*distance
msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth,
alert_data['ra'][i_obj],
alert_data['dec'][i_obj])
self.assertLess(distance_arcsec, 0.0005, msg=msg)
obs = obs_dict[alert_data['obshistId'][i_obj]]
chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
chipnum = int(chipname.replace('R', '').replace('S', '').
replace(' ', '').replace(';', '').replace(',', '').
replace(':', ''))
self.assertEqual(chipnum, alert_data['chipNum'][i_obj])
xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
pm_ra=self.pmra_truth[obj_dex],
pm_dec=self.pmdec_truth[obj_dex],
parallax=self.px_truth[obj_dex],
v_rad=self.vrad_truth[obj_dex],
obs_metadata=obs,
band=obs.bandpass)
self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4)
self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4)
dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj])
self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]],
dmag_sim, 3)
mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]]
m5 = obs.m5[mag_name]
q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj])
self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
q_mag, 4)
snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex],
bp_dict[mag_name],
self.obs_mag_cutoff[alert_data['band'][i_obj]],
phot_params)
self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4)
tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \
true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]]
snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name],
m5, phot_params)
self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4)
for val in obshistid_unqid_set:
self.assertIn(val, obshistid_unqid_simulated_set)
self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set))
astrometry_query = 'SELECT uniqueId, ra, dec, TAI '
astrometry_query += 'FROM baseline_astrometry'
astrometry_dtype = np.dtype([('uniqueId', int),
('ra', float),
('dec', float),
('TAI', float)])
tai_list = []
for obs in self.obs_list:
tai_list.append(obs.mjd.TAI)
tai_list = np.array(tai_list)
n_tot_ast_simulated = 0
for file_name in sqlite_file_list:
if not file_name.endswith('db'):
continue
full_name = os.path.join(self.output_dir, file_name)
self.assertTrue(os.path.exists(full_name))
alert_db = DBObject(full_name, driver='sqlite')
astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype)
if len(astrometry_data) == 0:
continue
mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI'])
for i_obj in range(len(astrometry_data)):
n_tot_ast_simulated += 1
obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1
ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex],
self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex],
self.px_truth[obj_dex], self.vrad_truth[obj_dex],
mjd=mjd_list[i_obj])
distance = angularSeparation(ra_truth, dec_truth,
astrometry_data['ra'][i_obj],
astrometry_data['dec'][i_obj])
self.assertLess(3600.0*distance, 0.0005)
del alert_gen
gc.collect()
self.assertGreater(n_tot_simulated, 10)
self.assertGreater(len(obshistid_unqid_simulated_set), 10)
self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations)
self.assertGreater(n_tot_ast_simulated, 0)
file_handle.write('rotskypos %.6f\n' % rot_sky)
file_handle.write('obshistid %d\n' % obshistid)
from lsst.sims.coordUtils import chipNameFromRaDecLSST
ra_list = np.array([ra_pointing, ra_pointing, ra_pointing + 0.5])
dec_list = np.array([dec_pointing, dec_pointing - 0.5, dec_pointing])
sed_name = 'starSED/kurucz/km30_5000.fits_g10_5040.gz'
with open('phosim_rot_control.txt', 'w') as control_file:
for ix, rot_sky in enumerate(np.arange(0.0, 271.0, 30.0)):
obs = ObservationMetaData(pointingRA=ra_pointing,
pointingDec=dec_pointing,
mjd=obs_pointing.mjd.TAI,
rotSkyPos=rot_sky)
file_name = 'coordutils_catalogs/phosim_rot_catalog_%.1f.txt' % rot_sky
with open(file_name, 'w') as output_file:
write_header(output_file, rot_sky, ix + 1)
for i_obj in range(len(ra_list)):
output_file.write(
'object %d %e %e 22.0 %s 0 0 0 0 0 0 point none none\n' %
(i_obj + 1, ra_list[i_obj], dec_list[i_obj], sed_name))
chip_names = chipNameFromRaDecLSST(ra_list, dec_list, obs_metadata=obs)
for i_obj in range(len(ra_list)):
control_file.write('%s %d %s\n' %
(file_name, i_obj + 1, chip_names[i_obj]))
control_file.write('\n')
def do_photometry(chunk, obs_lock, obs_metadata_dict, star_lock,
star_data_dict, job_lock, job_dict, out_dir):
"""
-Take a chunk from the stellar database
-Calculate the light curves for all of the stars in the chunk
-Write the light curves to disk
-Save the metadata and static data to dicts to be output later
Parameters
----------
chunk -- a set of stars returned by sqlite3.fetchmany()
the columns in chunk are:
simobjid, hpid, sedFilename, magNorm, ebv, varParamStr,
parallax, ra, decl
Note: chunk must be from one hpid (healpix pixel ID)
obs_lock -- the lock corresponding to the observation metadata
for this healpixel
obs_metadata_dict -- the dict where observation metadata is stored
star_lock -- the lock corresponding to the stellar metadata for this
healpixel
star_data_dict -- the dict where stellar metadata (i.e. static data) is
stored
job_lock -- the lock used to prevent too many processes from entering
light curve generation at once
job_dict -- the dict keeping track of how many jobs are doing light curve
generation
out_dir -- directory where light curve files are to be stored
Returns
-------
None
"""
dummy_sed = sims_photUtils.Sed()
# find the lookup file associating healpixel with obsHistID;
# this should probably actually be passed into the method
data_dir = '/astro/store/pogo4/danielsf/desc_dc2_truth'
assert os.path.isdir(data_dir)
hpid_lookup_name = os.path.join(data_dir, 'hpid_to_obsHistID_lookup.h5')
assert os.path.isfile(hpid_lookup_name)
metadata_dict = {}
metadata_keys = ['obsHistID', 'ra', 'dec', 'rotTelPos', 'mjd', 'filter']
hpid = int(chunk[0][1])
with h5py.File(hpid_lookup_name, 'r') as in_file:
valid_obsid = in_file['%d' % hpid][()]
for k in metadata_keys:
metadata_dict[k] = in_file[k][()]
valid_obsid = np.sort(valid_obsid)
valid_dex = np.searchsorted(metadata_dict['obsHistID'], valid_obsid)
np.testing.assert_array_equal(metadata_dict['obsHistID'][valid_dex],
valid_obsid)
for k in metadata_dict.keys():
metadata_dict[k] = metadata_dict[k][valid_dex]
# generate delta_magnitude light curves
has_dmag = False
while not has_dmag:
# make sure no more than 5 processes are doing this at once
with job_lock:
if job_dict['running_dmag'] >= 5:
continue
else:
job_dict['running_dmag'] += 1
#print('running dmag %d' % job_dict['running_dmag'])
t_start = time.time()
var_gen = VariabilityGenerator(chunk)
dmag_raw = var_gen.applyVariability(var_gen.varParamStr,
expmjd=metadata_dict['mjd'])
dmag_raw = dmag_raw.transpose([1, 2, 0])
# transpose so the columns are (star, mjd, filter)
assert dmag_raw.shape == (len(chunk), len(metadata_dict['mjd']), 6)
dmag = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=float)
for i_mjd in range(len(metadata_dict['mjd'])):
dmag[:, i_mjd] = dmag_raw[:, i_mjd, metadata_dict['filter'][i_mjd]]
del dmag_raw
has_dmag = True
with job_lock:
job_dict['running_dmag'] -= 1
#print('running dmag %d' % job_dict['running_dmag'])
quiescent_fluxes = np.zeros((len(chunk), 6), dtype=float)
for i_bp, bp in enumerate('ugrizy'):
quiescent_fluxes[:, i_bp] = dummy_sed.fluxFromMag(
var_gen.column_by_name('quiescent_%s' % bp))
t_dmag = time.time() - t_start
star_ra = np.array([c[7] for c in chunk])
star_dec = np.array([c[8] for c in chunk])
# Now figure out which obsHistID are observing which stars
obs_mask = np.zeros((len(chunk), len(metadata_dict['mjd'])), dtype=bool)
fov_radius = 2.1 # in degrees
t_start = time.time()
for i_obs in range(len(metadata_dict['mjd'])):
ra = np.degrees(metadata_dict['ra'][i_obs])
dec = np.degrees(metadata_dict['dec'][i_obs])
## back-of-the-envelope implementation
## (does not use focal plane model with chip gaps, etc.)
#ang_dist = angularSeparation(ra, dec, star_ra, star_dec)
#valid = ang_dist<fov_radius
rotTel = np.degrees(metadata_dict['rotTelPos'][i_obs])
obs_md = ObservationMetaData(pointingRA=ra,
pointingDec=dec,
mjd=metadata_dict['mjd'][i_obs])
rotSky = getRotSkyPos(ra, dec, obs_md, rotTel)
obs_md.rotSkyPos = rotSky
chip_name = chipNameFromRaDecLSST(star_ra,
star_dec,
obs_metadata=obs_md).astype(str)
valid = np.char.find(chip_name, 'None') < 0
obs_mask[:, i_obs] = valid
# verify that any stars with empty light curves are, in fact
# outside of DC2
region_of_interest = DC2_bounds()
xyz_offenders = []
for i_star in range(len(chunk)):
if obs_mask[i_star].sum() == 0:
xyz_offenders.append(
xyz_from_ra_dec(star_ra[i_star], star_dec[i_star]))
if len(xyz_offenders) > 0:
xyz_offenders = np.array(xyz_offenders)
v0 = region_of_interest.hs_list[0].contains_many_pts(xyz_offenders)
v1 = region_of_interest.hs_list[1].contains_many_pts(xyz_offenders)
v2 = region_of_interest.hs_list[2].contains_many_pts(xyz_offenders)
v3 = region_of_interest.hs_list[3].contains_many_pts(xyz_offenders)
if (v0 & v1 & v2 & v3).sum() > 0:
msg = "\n\nsome stars in healpixel %d unobserved\n\n" % hpid
raise RuntimeError(msg)
t_flux = time.time()
dflux_arr = []
for i_star in range(len(chunk)):
star_filter = metadata_dict['filter'][obs_mask[i_star]]
q_flux = quiescent_fluxes[i_star][star_filter]
assert len(q_flux) == obs_mask[i_star].sum()
q_mag = dummy_sed.magFromFlux(q_flux)
tot_mag = q_mag + dmag[i_star, obs_mask[i_star]]
tot_flux = dummy_sed.fluxFromMag(tot_mag)
dflux = tot_flux - q_flux
dflux_arr.append(dflux)
assert len(dflux) == obs_mask[i_star].sum()
# store metadata in the output dicts
with obs_lock:
obs_metadata_dict['mjd'].append(metadata_dict['mjd'])
obs_metadata_dict['obsHistID'].append(metadata_dict['obsHistID'])
obs_metadata_dict['filter'].append(metadata_dict['filter'])
with star_lock:
local_simobjid = var_gen.column_by_name('simobjid')
out_file_name = create_out_name(out_dir, hpid)
# write the light curves to disk now; otherwise, the memory
# footprint of the job becomes too large
with h5py.File(out_file_name, 'a') as out_file:
for i_star in range(len(local_simobjid)):
if len(dflux_arr[i_star]) == 0:
continue
simobjid = local_simobjid[i_star]
out_file.create_dataset('%d_flux' % simobjid,
data=dflux_arr[i_star])
out_file.create_dataset(
'%d_obsHistID' % simobjid,
data=metadata_dict['obsHistID'][obs_mask[i_star]])
star_data_dict['simobjid'].append(local_simobjid)
star_data_dict['ra'].append(star_ra)
star_data_dict['dec'].append(star_dec)
for i_bp, bp in enumerate('ugrizy'):
star_data_dict['quiescent_%s' % bp].append(quiescent_fluxes[:,
i_bp])