def dflux_for_kepler(chunk, filter_obs, mjd_obs, v_cache, dflux_out):
valid_obj = np.where(chunk['var_type']==1)
n_obj = len(valid_obj[0])
if n_obj==0:
return
params = {}
params['lc'] = chunk['lc_id'][valid_obj]
params['t0'] = chunk['t0'][valid_obj]
plc_model = ParametrizedLightCurveMixin()
dmag = plc_model.singleBandParametrizedLightCurve(np.array([True]*n_obj),
params, mjd_obs,
variability_cache=v_cache)
dummy_sed = Sed()
for i_bp, bp in enumerate('ugrizy'):
valid_obs = np.where(filter_obs==i_bp)
n_t = len(valid_obs[0])
if n_t==0:
continue
flux0 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj])
dmag_obs = dmag[:,valid_obs[0]].transpose()
assert dmag_obs.shape == (n_t, n_obj)
flux1 = dummy_sed.fluxFromMag(chunk['%smag' % bp][valid_obj]
+ dmag_obs)
dflux_local = (flux1-flux0).transpose()
for i_local, i_global in enumerate(valid_obj[0]):
dflux_out[i_global][valid_obs] = dflux_local[i_local]
def get_dmag(lc_id_list, out_file_name, seed, lock, v_cache):
rng = np.random.RandomState(seed)
plc = ParametrizedLightCurveMixin()
v_cache = create_variability_cache()
plc.load_parametrized_light_curves(variability_cache=v_cache)
dt = (60.0 * 10.0) / (3600.0 * 24.0)
mjd_base = np.arange(0.0, 180.0 + 2.0 * dt, dt)
dmag_abs_list = np.zeros(len(lc_id_list), dtype=float)
dmag_true_list = np.zeros(len(lc_id_list), dtype=float)
for i_lc, lc_int in enumerate(lc_id_list):
dmag_abs_max = -1.0
dmag_true_max = -1.0
for mjd_start in np.arange(59580.0, 59580.0 + 3654.0 + 181.0, 180.0):
mjd_arr = mjd_base + mjd_start
d_mjd = rng.random_sample(len(mjd_arr)) * 0.9 * dt
mjd_arr += d_mjd
q_flux, d_flux = plc._calc_dflux(lc_int,
mjd_arr,
variability_cache=v_cache)
d_flux = d_flux / q_flux
d_flux_abs_max = np.abs(d_flux).max()
dmag_abs_max_local = 2.5 * np.log10(1.0 + d_flux_abs_max)
if dmag_abs_max_local > dmag_abs_max:
dmag_abs_max = dmag_abs_max_local
d_flux_true_max = d_flux.max()
dmag_true_max_local = 2.5 * np.log10(1.0 + d_flux_true_max)
if dmag_true_max_local > dmag_true_max:
dmag_true_max = dmag_true_max_local
dmag_abs_list[i_lc] = dmag_abs_max
dmag_true_list[i_lc] = dmag_true_max
lock.acquire()
with open(out_file_name, 'a') as out_file:
for lc_id, dmag_abs, dmag_true in zip(lc_id_list, dmag_abs_list,
dmag_true_list):
out_file.write('%d %e %e\n' % (lc_id, dmag_abs, dmag_true))
lock.release()
def __init__(self,
opsimdb,
descqa_catalog,
dither=True,
min_mag=10,
minsource=100,
proper_motion=False,
protoDC2_ra=0,
protoDC2_dec=0,
star_db_name=None,
sed_lookup_dir=None,
agn_db_name=None,
agn_threads=1,
sn_db_name=None,
sprinkler=False,
host_image_dir=None,
host_data_dir=None,
config_dict=None,
gzip_threads=3,
objects_to_skip=()):
"""
Parameters
----------
opsimdb: str
OpSim db filename.
descqa_catalog: str
Name of the DESCQA galaxy catalog.
dither: bool [True]
Flag to enable the dithering included in the opsim db file.
min_mag: float [10]
Minimum value of the star magnitude at 500nm to include.
minsource: int [100]
Minimum number of objects for phosim.py to simulate a chip.
proper_motion: bool [True]
Flag to enable application of proper motion to stars.
protoDC2_ra: float [0]
Desired RA (J2000 degrees) of protoDC2 center.
protoDC2_dec: float [0]
Desired Dec (J2000 degrees) of protoDC2 center.
star_db_name: str [None]
Filename of the database containing stellar sources
sed_lookup_dir: str [None]
Directory where the SED lookup tables reside.
agn_db_name: str [None]
Filename of the agn parameter sqlite db file.
agn_threads: int [1]
Number of threads to use when simulating AGN variability
sn_db_name: str [None]
Filename of the supernova parameter sqlite db file.
sprinkler: bool [False]
Flag to enable the Sprinkler.
host_image_dir: string
The location of the FITS images of lensed AGN/SNe hosts produced by generate_lensed_hosts_***.py
host_data_dir: string
Location of csv file of lensed host data created by the sprinkler
gzip_threads: int
The number of gzip jobs that can be started in parallel after
catalogs are written (default=3)
objects_to_skip: set-like or list-like [()]
Collection of object types to skip, e.g., stars, knots, bulges, disks, sne, agn
"""
self.t_start = time.time()
if not os.path.exists(opsimdb):
raise RuntimeError('%s does not exist' % opsimdb)
self.gzip_threads = gzip_threads
# load the data for the parametrized light
# curve stellar variability model into a
# global cache
plc = ParametrizedLightCurveMixin()
plc.load_parametrized_light_curves()
self.config_dict = config_dict if config_dict is not None else {}
self.descqa_catalog = descqa_catalog
self.dither = dither
self.min_mag = min_mag
self.minsource = minsource
self.proper_motion = proper_motion
self.protoDC2_ra = protoDC2_ra
self.protoDC2_dec = protoDC2_dec
self.phot_params = PhotometricParameters(nexp=1, exptime=30)
self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
self.obs_gen = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
if star_db_name is None:
raise IOError("Need to specify star_db_name")
if not os.path.isfile(star_db_name):
raise IOError("%s is not a file\n" % star_db_name +
"(This is what you specified for star_db_name")
self.star_db = DC2StarObj(database=star_db_name, driver='sqlite')
self.sprinkler = sprinkler
if self.sprinkler and not HAS_TWINKLES:
raise RuntimeError("You are trying to enable the sprinkler; "
"but Twinkles cannot be imported")
if not os.path.isdir(sed_lookup_dir):
raise IOError("\n%s\nis not a dir" % sed_lookup_dir)
self.sed_lookup_dir = sed_lookup_dir
self._agn_threads = agn_threads
if agn_db_name is not None:
if os.path.exists(agn_db_name):
self.agn_db_name = agn_db_name
else:
raise IOError("Path to Proto DC2 AGN database does not exist.")
else:
self.agn_db_name = None
self.sn_db_name = None
if sn_db_name is not None:
if os.path.isfile(sn_db_name):
self.sn_db_name = sn_db_name
else:
raise IOError("%s is not a file" % sn_db_name)
if host_image_dir is None and self.sprinkler is not False:
raise IOError(
"Need to specify the name of the host image directory.")
elif self.sprinkler is not False:
if os.path.exists(host_image_dir):
self.host_image_dir = host_image_dir
else:
raise IOError("Path to host image directory" +
"\n\n%s\n\n" % host_image_dir +
"does not exist.")
if host_data_dir is None and self.sprinkler is not False:
raise IOError(
"Need to specify the name of the host data directory.")
elif self.sprinkler is not False:
if os.path.exists(host_data_dir):
self.host_data_dir = host_data_dir
else:
raise IOError(
"Path to host data directory does not exist.\n\n",
"%s\n\n" % host_data_dir)
self.instcats = get_instance_catalogs()
object_types = 'stars knots bulges disks sprinkled hosts sne agn'.split(
)
if any([_ not in object_types for _ in objects_to_skip]):
raise RuntimeError(f'objects_to_skip ({objects_to_skip}) '
'contains invalid object types')
self.do_obj_type = {_: _ not in objects_to_skip for _ in object_types}
def __init__(self,
opsimdb,
descqa_catalog,
dither=True,
min_mag=10,
minsource=100,
proper_motion=False,
imsim_catalog=False,
protoDC2_ra=0,
protoDC2_dec=0,
agn_db_name=None,
sprinkler=False):
"""
Parameters
----------
obsimdb: str
OpSim db filename.
descqa_catalog: str
Name of the DESCQA galaxy catalog.
dither: bool [True]
Flag to enable the dithering included in the opsim db file.
min_mag: float [10]
Minimum value of the star magnitude at 500nm to include.
minsource: int [100]
Minimum number of objects for phosim.py to simulate a chip.
proper_motion: bool [True]
Flag to enable application of proper motion to stars.
imsim_catalog: bool [False]
Flag to write an imsim-style object catalog.
protoDC2_ra: float [0]
Desired RA (J2000 degrees) of protoDC2 center.
protoDC2_dec: float [0]
Desired Dec (J2000 degrees) of protoDC2 center.
agn_db_name: str [None]
Filename of the agn parameter sqlite db file.
sprinkler: bool [False]
Flag to enable the Sprinkler.
"""
if not os.path.exists(opsimdb):
raise RuntimeError('%s does not exist' % opsimdb)
# load the data for the parametrized light
# curve stellar variability model into a
# global cache
plc = ParametrizedLightCurveMixin()
plc.load_parametrized_light_curves()
self.descqa_catalog = descqa_catalog
self.dither = dither
self.min_mag = min_mag
self.minsource = minsource
self.proper_motion = proper_motion
self.imsim_catalog = imsim_catalog
self.protoDC2_ra = protoDC2_ra
self.protoDC2_dec = protoDC2_dec
self.phot_params = PhotometricParameters(nexp=1, exptime=30)
self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles()
self.obs_gen = ObservationMetaDataGenerator(database=opsimdb,
driver='sqlite')
self.star_db = StarObj(database='LSSTCATSIM',
host='fatboy.phys.washington.edu',
port=1433,
driver='mssql+pymssql')
if agn_db_name is None:
raise IOError("Need to specify an Proto DC2 AGN database.")
else:
if os.path.exists(agn_db_name):
self.agn_db_name = agn_db_name
else:
raise IOError("Path to Proto DC2 AGN database does not exist.")
self.sprinkler = sprinkler
self.instcats = get_instance_catalogs(imsim_catalog)
if isinstance(args.htmid, numbers.Number):
htmid_list = [args.htmid]
else:
htmid_list = args.htmid
else:
htmid_list = []
with open(args.htmid_file, 'r') as in_file:
for line in in_file:
if line.startswith('#'):
continue
params = line.strip().split()
htmid_list.append(int(params[0]))
variability_cache = create_variability_cache()
plc = ParametrizedLightCurveMixin()
plc.load_parametrized_light_curves(variability_cache=variability_cache)
variability_cache['rrly_map'] = {}
lc_dir = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], 'rrly_lc')
assert os.path.isdir(lc_dir)
with open('data/rrly_lc_map.txt', 'r') as in_file:
for line in in_file:
p = line.strip().split(';')
variability_cache['rrly_map'][int(p[0])] = os.path.join(lc_dir, p[1])
m5_single = {}
with open(args.m5_single, 'r') as in_file:
for line in in_file:
if line.startswith('#'):
continue
def test_ParametrizedLightCurve_in_catalog(self):
"""
Test the performance of applyParametrizedLightCurve()
in the context of an InstanceCatalog
"""
# Create dummy light curve parameters
lc_temp_file_name = tempfile.mktemp(
prefix='test_ParametrizedLightCurve_in_catalog', suffix='.gz')
rng = np.random.RandomState(1621145)
n_c_1 = 10
a1_list = rng.random_sample(n_c_1) * 5.0
b1_list = (rng.random_sample(n_c_1) - 0.5) * 2.0
c1_list = (rng.random_sample(n_c_1) - 0.5) * 0.1
omega1_list = rng.random_sample(n_c_1) * 20.0
tau1_list = rng.random_sample(n_c_1) * 100.0
median1 = 100.0
n_c_2 = 15
a2_list = rng.random_sample(n_c_2) * 5.0
b2_list = (rng.random_sample(n_c_2) - 0.5) * 2.0
c2_list = (rng.random_sample(n_c_2) - 0.5) * 0.1
omega2_list = rng.random_sample(n_c_2) * 20.0
tau2_list = rng.random_sample(n_c_2) * 100.0
median2 = 200.0
with gzip.open(lc_temp_file_name, 'w') as out_file:
out_file.write(b'# a header\n')
out_file.write(b'kplr999990000_lc.txt 100 1.0e+02 %d ' % n_c_1)
for i_c in range(n_c_1):
out_file.write(b'%e ' % (1.0 / (i_c + 1)))
out_file.write(b'%e ' % median1)
for i_c in range(n_c_1):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a1_list[i_c], b1_list[i_c], c1_list[i_c],
omega1_list[i_c], tau1_list[i_c]))
out_file.write(b'\n')
out_file.write(b'kplr999990001_lc.txt 100 1.0e+02 %d ' % n_c_2)
for i_c in range(n_c_2):
out_file.write(b'%e ' % (1.0 / (i_c + 1)))
out_file.write(b'%e ' % median2)
for i_c in range(n_c_2):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a2_list[i_c], b2_list[i_c], c2_list[i_c],
omega2_list[i_c], tau2_list[i_c]))
out_file.write(b'\n')
# Create dummy database of astrophysical sources
db_temp_file_name = tempfile.mktemp(
prefix='test_ParametrizedLightCurve_in_catalog_db', suffix='.txt')
lc_list = [999990001, None, 999990001, 999990000]
t0_list = [1729.1, None, 2345.1, 10.9]
with open(db_temp_file_name, 'w') as out_file:
out_file.write('# a header\n')
for i_obj in range(len(lc_list)):
if lc_list[i_obj] is not None:
paramStr = '{"m":"kplr", "p":{"lc":%d, "t0":%.3f}}' % (
lc_list[i_obj], t0_list[i_obj])
else:
paramStr = None
out_file.write('%d;10.0;20.0;0.01;0.01;%s\n' %
(i_obj, paramStr))
dtype = np.dtype([('simobjid', int), ('ra', float), ('dec', float),
('ebv', float), ('parallax', float),
('varParamStr', str, 100)])
db = fileDBObject(db_temp_file_name,
runtable='test',
dtype=dtype,
delimiter=';',
idColKey='simobjid')
class ParametrizedVarParamStrCat(InstanceCatalog, VariabilityStars):
column_outputs = [
'simobjid', 'delta_lsst_u', 'delta_lsst_g', 'delta_lsst_r',
'delta_lsst_i', 'delta_lsst_z', 'delta_lsst_y'
]
default_formats = {'f': '%.15g'}
obs = ObservationMetaData(mjd=59580.0)
cat = ParametrizedVarParamStrCat(db, obs_metadata=obs)
cat.load_parametrized_light_curves(lc_temp_file_name)
cat_out_name = tempfile.mktemp(
prefix='test_ParametrizedLightCurve_in_cat_out', suffix='.txt')
cat.write_catalog(cat_out_name)
kp = ParametrizedLightCurveMixin()
cat_dtype = np.dtype([('simobjid', int), ('du', float), ('dg', float),
('dr', float), ('di', float), ('dz', float),
('dy', float)])
cat_data = np.genfromtxt(cat_out_name, dtype=cat_dtype, delimiter=', ')
for i_obj in range(len(cat_data)):
obj_id = cat_data['simobjid'][i_obj]
if lc_list[obj_id] is None:
self.assertEqual(cat_data['du'][i_obj], 0.0)
self.assertEqual(cat_data['dg'][i_obj], 0.0)
self.assertEqual(cat_data['dr'][i_obj], 0.0)
self.assertEqual(cat_data['di'][i_obj], 0.0)
self.assertEqual(cat_data['dz'][i_obj], 0.0)
self.assertEqual(cat_data['dy'][i_obj], 0.0)
else:
q_flux, d_flux = kp._calc_dflux(lc_list[obj_id],
obs.mjd.TAI - t0_list[obj_id])
d_mag_true = -2.5 * np.log10(1.0 + d_flux / q_flux)
self.assertGreater(np.abs(d_mag_true), 0.0001)
self.assertAlmostEqual(cat_data['du'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dg'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dr'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['di'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dz'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dy'][i_obj], d_mag_true, 15)
if os.path.exists(cat_out_name):
os.unlink(cat_out_name)
if os.path.exists(db_temp_file_name):
os.unlink(db_temp_file_name)
sims_clean_up()
if os.path.exists(lc_temp_file_name):
os.unlink(lc_temp_file_name)
def test_applyParametrizedLightCurve_manyExpmjd(self):
"""
test applyParametrizedLightCurve on an array of expmjd values
by creating a dummy light curve file with known
parameters, generating magnitudes, and comparing to
the expected outputs.
We will use _calc_dflux() to calculate the known truth,
since that method was tested in test_calc_dflux()
"""
lc_temp_file_name = tempfile.mktemp(
prefix='test_applyParametrizedLightCurve_manyexpmjd', suffix='.gz')
rng = np.random.RandomState(13291)
n_c_1 = 10
a1_list = rng.random_sample(n_c_1) * 5.0
b1_list = (rng.random_sample(n_c_1) - 0.5) * 2.0
c1_list = (rng.random_sample(n_c_1) - 0.5) * 0.1
omega1_list = rng.random_sample(n_c_1) * 20.0
tau1_list = rng.random_sample(n_c_1) * 100.0
median1 = 100.0
n_c_2 = 15
a2_list = rng.random_sample(n_c_2) * 5.0
b2_list = (rng.random_sample(n_c_2) - 0.5) * 2.0
c2_list = (rng.random_sample(n_c_2) - 0.5) * 0.1
omega2_list = rng.random_sample(n_c_2) * 20.0
tau2_list = rng.random_sample(n_c_2) * 100.0
median2 = 200.0
with gzip.open(lc_temp_file_name, 'w') as out_file:
out_file.write(b'# a header\n')
out_file.write(b'kplr999900000_lc.txt 100 1.0e+02 %d ' % n_c_1)
for i_c in range(n_c_1):
out_file.write(b'%e ' % (1.0 / (i_c + 1)))
out_file.write(b'%e ' % median1)
for i_c in range(n_c_1):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a1_list[i_c], b1_list[i_c], c1_list[i_c],
omega1_list[i_c], tau1_list[i_c]))
out_file.write(b'\n')
out_file.write(b'kplr999900001_lc.txt 100 1.0e+02 %d ' % n_c_2)
for i_c in range(n_c_2):
out_file.write(b'%e ' % (1.0 / (i_c + 1)))
out_file.write(b'%e ' % median2)
for i_c in range(n_c_2):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a2_list[i_c], b2_list[i_c], c2_list[i_c],
omega2_list[i_c], tau2_list[i_c]))
out_file.write(b'\n')
params = {}
params['lc'] = np.array([999900001, 999900000, None, 999900001])
params['t0'] = np.array([223.1, 1781.45, None, 32.0])
kp = ParametrizedLightCurveMixin()
kp.load_parametrized_light_curves(lc_temp_file_name)
# first test that passing in an empty set of params
# results in an empty numpy array (so that the 'dry
# run' of catalog generation does not fail)
d_mag_out = kp.applyParametrizedLightCurve([], {}, 1.0)
np.testing.assert_array_equal(d_mag_out,
np.array([[], [], [], [], [], []]))
expmjd = rng.random_sample(10) * 10000.0 + 59580.0
d_mag_out = kp.applyParametrizedLightCurve([], params, expmjd)
self.assertEqual(d_mag_out.shape, (6, 4, 10))
for i_obj in range(4):
if i_obj == 2:
for i_filter in range(6):
np.testing.assert_array_equal(d_mag_out[i_filter][i_obj],
np.zeros(10))
else:
q_flux, d_flux = kp._calc_dflux(params['lc'][i_obj],
expmjd - params['t0'][i_obj])
d_mag_truth = -2.5 * np.log10(1.0 + d_flux / q_flux)
nan_vals = np.where(np.isnan(d_mag_truth))
self.assertEqual(len(nan_vals[0]), 0)
for i_filter in range(6):
np.testing.assert_array_equal(d_mag_out[i_filter][i_obj],
d_mag_truth)
sims_clean_up()
if os.path.exists(lc_temp_file_name):
os.unlink(lc_temp_file_name)
def test_calc_dflux(self):
"""
Test the method that calculates the flux of
parametrized light curves by generating a fake light
curve library with known parameters, calculating
the fluxes, and comparing to the expected results.
"""
lc_temp_file_name = tempfile.mktemp(prefix='test_calc_dflux_lc',
suffix='.gz')
rng = np.random.RandomState(7124)
n_c_1 = 10
a1_list = rng.random_sample(n_c_1) * 5.0
b1_list = (rng.random_sample(n_c_1) - 0.5) * 2.0
c1_list = (rng.random_sample(n_c_1) - 0.5) * 0.1
omega1_list = rng.random_sample(n_c_1) * 20.0
tau1_list = rng.random_sample(n_c_1) * 100.0
median1 = 100.0
n_c_2 = 15
a2_list = rng.random_sample(n_c_2) * 5.0
b2_list = (rng.random_sample(n_c_2) - 0.5) * 2.0
c2_list = (rng.random_sample(n_c_2) - 0.5) * 0.1
omega2_list = rng.random_sample(n_c_2) * 20.0
tau2_list = rng.random_sample(n_c_2) * 100.0
median2 = 200.0
with gzip.open(lc_temp_file_name, 'w') as out_file:
out_file.write(b'# a header\n')
out_file.write(b'kplr990000000_lc.txt 100 1.0e+02 %d ' % n_c_1)
for i_c in range(n_c_1):
out_file.write(b'%e ' % (1.0 / (i_c + 1)))
out_file.write(b'%e ' % median1)
for i_c in range(n_c_1):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a1_list[i_c], b1_list[i_c], c1_list[i_c],
omega1_list[i_c], tau1_list[i_c]))
out_file.write(b'\n')
out_file.write(b'kplr990000001_lc.txt 100 1.0e+02 %d ' % n_c_2)
for i_c in range(n_c_2):
out_file.write(b'%e ' % (1.0 / (i_c + 1)))
out_file.write(b'%e ' % median2)
for i_c in range(n_c_2):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a2_list[i_c], b2_list[i_c], c2_list[i_c],
omega2_list[i_c], tau2_list[i_c]))
out_file.write(b'\n')
expmjd = rng.random_sample(100) * 200.0
kp = ParametrizedLightCurveMixin()
kp.load_parametrized_light_curves(lc_temp_file_name)
q_flux, d_flux = kp._calc_dflux(990000000, expmjd)
self.assertAlmostEqual(q_flux, median1 + c1_list.sum(), 10)
true_flux = np.zeros(len(expmjd))
for i_c in range(n_c_1):
arg = omega1_list[i_c] * (expmjd - tau1_list[i_c])
true_flux += a1_list[i_c] * np.cos(arg)
true_flux += b1_list[i_c] * np.sin(arg)
self.assertEqual(len(d_flux), len(true_flux))
np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10)
q_flux, d_flux = kp._calc_dflux(990000001, expmjd)
self.assertAlmostEqual(q_flux, median2 + c2_list.sum(), 10)
true_flux = np.zeros(len(expmjd))
for i_c in range(n_c_2):
arg = omega2_list[i_c] * (expmjd - tau2_list[i_c])
true_flux += a2_list[i_c] * np.cos(arg)
true_flux += b2_list[i_c] * np.sin(arg)
self.assertEqual(len(d_flux), len(true_flux))
np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10)
sims_clean_up()
if os.path.exists(lc_temp_file_name):
os.unlink(lc_temp_file_name)
for chunk in data_iter:
for star in chunk:
param_dict = json.loads(star['varParamStr'])
lc_id = param_dict['p']['lc']
out_file.write('%d %d %d\n' % (star['htmid'], star['simobjid'], dmag_dict[lc_id]))
if __name__ == "__main__":
_out_dir = os.path.join('/astro/store/pogo4/danielsf/dmag_max_data')
_out_dir = '.'
if not os.path.isdir(_out_dir):
raise RuntimeError('%s is not a dir' % _out_dir)
plc = ParametrizedLightCurveMixin()
variability_cache = create_variability_cache()
plc.load_parametrized_light_curves(variability_cache=variability_cache)
time_arr = np.arange(0.0, 3653.0, 0.1)
dmag_dict = {}
t_start = time.time()
for lc_id in variability_cache['_PARAMETRIZED_LC_MODELS']:
q_flux, d_flux = plc._calc_dflux(lc_id, time_arr,
variability_cache=variability_cache)
d_flux = np.where(d_flux>-q_flux, d_flux, -0.9*q_flux)
try:
dmag = 2.5*np.log10(1.0+d_flux/q_flux)
dmag = np.where(np.abs(dmag)>0.0, dmag, 1.0e-20)
dmag = np.where(np.abs(dmag)<1.0e10, dmag, 1.0e10)
bad = np.where(np.logical_or(np.isnan(dmag),np.isinf(dmag)))
def test_ParametrizedLightCurve_in_catalog(self):
"""
Test the performance of applyParametrizedLightCurve()
in the context of an InstanceCatalog
"""
# Create dummy light curve parameters
lc_temp_file_name = tempfile.mktemp(prefix='test_ParametrizedLightCurve_in_catalog',
suffix='.gz')
rng = np.random.RandomState(1621145)
n_c_1 = 10
a1_list = rng.random_sample(n_c_1)*5.0
b1_list = (rng.random_sample(n_c_1)-0.5)*2.0
c1_list = (rng.random_sample(n_c_1)-0.5)*0.1
omega1_list = rng.random_sample(n_c_1)*20.0
tau1_list = rng.random_sample(n_c_1)*100.0
median1 = 100.0
n_c_2 = 15
a2_list = rng.random_sample(n_c_2)*5.0
b2_list = (rng.random_sample(n_c_2)-0.5)*2.0
c2_list = (rng.random_sample(n_c_2)-0.5)*0.1
omega2_list = rng.random_sample(n_c_2)*20.0
tau2_list = rng.random_sample(n_c_2)*100.0
median2 = 200.0
with gzip.open(lc_temp_file_name, 'w') as out_file:
out_file.write(b'# a header\n')
out_file.write(b'kplr999990000_lc.txt 100 1.0e+02 %d ' % n_c_1)
for i_c in range(n_c_1):
out_file.write(b'%e ' % (1.0/(i_c+1)))
out_file.write(b'%e ' % median1)
for i_c in range(n_c_1):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a1_list[i_c], b1_list[i_c], c1_list[i_c],
omega1_list[i_c], tau1_list[i_c]))
out_file.write(b'\n')
out_file.write(b'kplr999990001_lc.txt 100 1.0e+02 %d ' % n_c_2)
for i_c in range(n_c_2):
out_file.write(b'%e ' % (1.0/(i_c+1)))
out_file.write(b'%e ' % median2)
for i_c in range(n_c_2):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a2_list[i_c], b2_list[i_c], c2_list[i_c],
omega2_list[i_c], tau2_list[i_c]))
out_file.write(b'\n')
# Create dummy database of astrophysical sources
db_temp_file_name = tempfile.mktemp(prefix='test_ParametrizedLightCurve_in_catalog_db',
suffix='.txt')
lc_list = [999990001, None, 999990001, 999990000]
t0_list = [1729.1, None, 2345.1, 10.9]
with open(db_temp_file_name, 'w') as out_file:
out_file.write('# a header\n')
for i_obj in range(len(lc_list)):
if lc_list[i_obj] is not None:
paramStr = '{"m":"kplr", "p":{"lc":%d, "t0":%.3f}}' % (lc_list[i_obj], t0_list[i_obj])
else:
paramStr = None
out_file.write('%d;10.0;20.0;0.01;0.01;%s\n' % (i_obj, paramStr))
dtype = np.dtype([('simobjid', int), ('ra', float), ('dec', float),
('ebv', float), ('parallax', float), ('varParamStr', str, 100)])
db = fileDBObject(db_temp_file_name, runtable='test', dtype=dtype, delimiter=';',
idColKey='simobjid')
class ParametrizedVarParamStrCat(InstanceCatalog, VariabilityStars):
column_outputs = ['simobjid', 'delta_lsst_u', 'delta_lsst_g', 'delta_lsst_r',
'delta_lsst_i', 'delta_lsst_z', 'delta_lsst_y']
default_formats = {'f':'%.15g'}
obs = ObservationMetaData(mjd=59580.0)
cat = ParametrizedVarParamStrCat(db, obs_metadata=obs)
cat.load_parametrized_light_curves(lc_temp_file_name)
cat_out_name = tempfile.mktemp(prefix='test_ParametrizedLightCurve_in_cat_out',
suffix='.txt')
cat.write_catalog(cat_out_name)
kp = ParametrizedLightCurveMixin()
cat_dtype = np.dtype([('simobjid', int), ('du', float), ('dg', float),
('dr', float), ('di', float), ('dz', float),
('dy', float)])
cat_data = np.genfromtxt(cat_out_name, dtype=cat_dtype, delimiter=', ')
for i_obj in range(len(cat_data)):
obj_id = cat_data['simobjid'][i_obj]
if lc_list[obj_id] is None:
self.assertEqual(cat_data['du'][i_obj], 0.0)
self.assertEqual(cat_data['dg'][i_obj], 0.0)
self.assertEqual(cat_data['dr'][i_obj], 0.0)
self.assertEqual(cat_data['di'][i_obj], 0.0)
self.assertEqual(cat_data['dz'][i_obj], 0.0)
self.assertEqual(cat_data['dy'][i_obj], 0.0)
else:
q_flux, d_flux = kp._calc_dflux(lc_list[obj_id], obs.mjd.TAI-t0_list[obj_id])
d_mag_true = -2.5*np.log10(1.0+d_flux/q_flux)
self.assertGreater(np.abs(d_mag_true), 0.0001)
self.assertAlmostEqual(cat_data['du'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dg'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dr'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['di'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dz'][i_obj], d_mag_true, 15)
self.assertAlmostEqual(cat_data['dy'][i_obj], d_mag_true, 15)
if os.path.exists(cat_out_name):
os.unlink(cat_out_name)
if os.path.exists(db_temp_file_name):
os.unlink(db_temp_file_name)
sims_clean_up()
if os.path.exists(lc_temp_file_name):
os.unlink(lc_temp_file_name)
def test_applyParametrizedLightCurve_manyExpmjd(self):
"""
test applyParametrizedLightCurve on an array of expmjd values
by creating a dummy light curve file with known
parameters, generating magnitudes, and comparing to
the expected outputs.
We will use _calc_dflux() to calculate the known truth,
since that method was tested in test_calc_dflux()
"""
lc_temp_file_name = tempfile.mktemp(prefix='test_applyParametrizedLightCurve_manyexpmjd',
suffix='.gz')
rng = np.random.RandomState(13291)
n_c_1 = 10
a1_list = rng.random_sample(n_c_1)*5.0
b1_list = (rng.random_sample(n_c_1)-0.5)*2.0
c1_list = (rng.random_sample(n_c_1)-0.5)*0.1
omega1_list = rng.random_sample(n_c_1)*20.0
tau1_list = rng.random_sample(n_c_1)*100.0
median1 = 100.0
n_c_2 = 15
a2_list = rng.random_sample(n_c_2)*5.0
b2_list = (rng.random_sample(n_c_2)-0.5)*2.0
c2_list = (rng.random_sample(n_c_2)-0.5)*0.1
omega2_list = rng.random_sample(n_c_2)*20.0
tau2_list = rng.random_sample(n_c_2)*100.0
median2 = 200.0
with gzip.open(lc_temp_file_name, 'w') as out_file:
out_file.write(b'# a header\n')
out_file.write(b'kplr999900000_lc.txt 100 1.0e+02 %d ' % n_c_1)
for i_c in range(n_c_1):
out_file.write(b'%e ' % (1.0/(i_c+1)))
out_file.write(b'%e ' % median1)
for i_c in range(n_c_1):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a1_list[i_c], b1_list[i_c], c1_list[i_c],
omega1_list[i_c], tau1_list[i_c]))
out_file.write(b'\n')
out_file.write(b'kplr999900001_lc.txt 100 1.0e+02 %d ' % n_c_2)
for i_c in range(n_c_2):
out_file.write(b'%e ' % (1.0/(i_c+1)))
out_file.write(b'%e ' % median2)
for i_c in range(n_c_2):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a2_list[i_c], b2_list[i_c], c2_list[i_c],
omega2_list[i_c], tau2_list[i_c]))
out_file.write(b'\n')
params = {}
params['lc'] = np.array([999900001, 999900000, None, 999900001])
params['t0'] = np.array([223.1, 1781.45, None, 32.0])
kp = ParametrizedLightCurveMixin()
kp.load_parametrized_light_curves(lc_temp_file_name)
# first test that passing in an empty set of params
# results in an empty numpy array (so that the 'dry
# run' of catalog generation does not fail)
d_mag_out = kp.applyParametrizedLightCurve([],{},1.0)
np.testing.assert_array_equal(d_mag_out,
np.array([[],[],[],[],[],[]]))
expmjd = rng.random_sample(10)*10000.0 + 59580.0
d_mag_out = kp.applyParametrizedLightCurve([], params, expmjd)
self.assertEqual(d_mag_out.shape, (6, 4, 10))
for i_obj in range(4):
if i_obj == 2:
for i_filter in range(6):
np.testing.assert_array_equal(d_mag_out[i_filter][i_obj],
np.zeros(10))
else:
q_flux, d_flux = kp._calc_dflux(params['lc'][i_obj],
expmjd-params['t0'][i_obj])
d_mag_truth = -2.5*np.log10(1.0+d_flux/q_flux)
nan_vals = np.where(np.isnan(d_mag_truth))
self.assertEqual(len(nan_vals[0]), 0)
for i_filter in range(6):
np.testing.assert_array_equal(d_mag_out[i_filter][i_obj], d_mag_truth)
sims_clean_up()
if os.path.exists(lc_temp_file_name):
os.unlink(lc_temp_file_name)
def test_calc_dflux(self):
"""
Test the method that calculates the flux of
parametrized light curves by generating a fake light
curve library with known parameters, calculating
the fluxes, and comparing to the expected results.
"""
lc_temp_file_name = tempfile.mktemp(prefix='test_calc_dflux_lc',
suffix='.gz')
rng = np.random.RandomState(7124)
n_c_1 = 10
a1_list = rng.random_sample(n_c_1)*5.0
b1_list = (rng.random_sample(n_c_1)-0.5)*2.0
c1_list = (rng.random_sample(n_c_1)-0.5)*0.1
omega1_list = rng.random_sample(n_c_1)*20.0
tau1_list = rng.random_sample(n_c_1)*100.0
median1 = 100.0
n_c_2 = 15
a2_list = rng.random_sample(n_c_2)*5.0
b2_list = (rng.random_sample(n_c_2)-0.5)*2.0
c2_list = (rng.random_sample(n_c_2)-0.5)*0.1
omega2_list = rng.random_sample(n_c_2)*20.0
tau2_list = rng.random_sample(n_c_2)*100.0
median2 = 200.0
with gzip.open(lc_temp_file_name, 'w') as out_file:
out_file.write(b'# a header\n')
out_file.write(b'kplr990000000_lc.txt 100 1.0e+02 %d ' % n_c_1)
for i_c in range(n_c_1):
out_file.write(b'%e ' % (1.0/(i_c+1)))
out_file.write(b'%e ' % median1)
for i_c in range(n_c_1):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a1_list[i_c], b1_list[i_c], c1_list[i_c],
omega1_list[i_c], tau1_list[i_c]))
out_file.write(b'\n')
out_file.write(b'kplr990000001_lc.txt 100 1.0e+02 %d ' % n_c_2)
for i_c in range(n_c_2):
out_file.write(b'%e ' % (1.0/(i_c+1)))
out_file.write(b'%e ' % median2)
for i_c in range(n_c_2):
out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' %
(a2_list[i_c], b2_list[i_c], c2_list[i_c],
omega2_list[i_c], tau2_list[i_c]))
out_file.write(b'\n')
expmjd = rng.random_sample(100)*200.0
kp = ParametrizedLightCurveMixin()
kp.load_parametrized_light_curves(lc_temp_file_name)
q_flux, d_flux = kp._calc_dflux(990000000, expmjd)
self.assertAlmostEqual(q_flux, median1+c1_list.sum(), 10)
true_flux = np.zeros(len(expmjd))
for i_c in range(n_c_1):
arg = omega1_list[i_c]*(expmjd-tau1_list[i_c])
true_flux += a1_list[i_c]*np.cos(arg)
true_flux += b1_list[i_c]*np.sin(arg)
self.assertEqual(len(d_flux), len(true_flux))
np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10)
q_flux, d_flux = kp._calc_dflux(990000001, expmjd)
self.assertAlmostEqual(q_flux, median2+c2_list.sum(), 10)
true_flux = np.zeros(len(expmjd))
for i_c in range(n_c_2):
arg = omega2_list[i_c]*(expmjd-tau2_list[i_c])
true_flux += a2_list[i_c]*np.cos(arg)
true_flux += b2_list[i_c]*np.sin(arg)
self.assertEqual(len(d_flux), len(true_flux))
np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10)
sims_clean_up()
if os.path.exists(lc_temp_file_name):
os.unlink(lc_temp_file_name)