def get_dmag(lc_id_list, out_file_name, seed, lock, v_cache): rng = np.random.RandomState(seed) plc = ParametrizedLightCurveMixin() v_cache = create_variability_cache() plc.load_parametrized_light_curves(variability_cache=v_cache) dt = (60.0 * 10.0) / (3600.0 * 24.0) mjd_base = np.arange(0.0, 180.0 + 2.0 * dt, dt) dmag_abs_list = np.zeros(len(lc_id_list), dtype=float) dmag_true_list = np.zeros(len(lc_id_list), dtype=float) for i_lc, lc_int in enumerate(lc_id_list): dmag_abs_max = -1.0 dmag_true_max = -1.0 for mjd_start in np.arange(59580.0, 59580.0 + 3654.0 + 181.0, 180.0): mjd_arr = mjd_base + mjd_start d_mjd = rng.random_sample(len(mjd_arr)) * 0.9 * dt mjd_arr += d_mjd q_flux, d_flux = plc._calc_dflux(lc_int, mjd_arr, variability_cache=v_cache) d_flux = d_flux / q_flux d_flux_abs_max = np.abs(d_flux).max() dmag_abs_max_local = 2.5 * np.log10(1.0 + d_flux_abs_max) if dmag_abs_max_local > dmag_abs_max: dmag_abs_max = dmag_abs_max_local d_flux_true_max = d_flux.max() dmag_true_max_local = 2.5 * np.log10(1.0 + d_flux_true_max) if dmag_true_max_local > dmag_true_max: dmag_true_max = dmag_true_max_local dmag_abs_list[i_lc] = dmag_abs_max dmag_true_list[i_lc] = dmag_true_max lock.acquire() with open(out_file_name, 'a') as out_file: for lc_id, dmag_abs, dmag_true in zip(lc_id_list, dmag_abs_list, dmag_true_list): out_file.write('%d %e %e\n' % (lc_id, dmag_abs, dmag_true)) lock.release()
def __init__(self, opsimdb, descqa_catalog, dither=True, min_mag=10, minsource=100, proper_motion=False, protoDC2_ra=0, protoDC2_dec=0, star_db_name=None, sed_lookup_dir=None, agn_db_name=None, agn_threads=1, sn_db_name=None, sprinkler=False, host_image_dir=None, host_data_dir=None, config_dict=None, gzip_threads=3, objects_to_skip=()): """ Parameters ---------- opsimdb: str OpSim db filename. descqa_catalog: str Name of the DESCQA galaxy catalog. dither: bool [True] Flag to enable the dithering included in the opsim db file. min_mag: float [10] Minimum value of the star magnitude at 500nm to include. minsource: int [100] Minimum number of objects for phosim.py to simulate a chip. proper_motion: bool [True] Flag to enable application of proper motion to stars. protoDC2_ra: float [0] Desired RA (J2000 degrees) of protoDC2 center. protoDC2_dec: float [0] Desired Dec (J2000 degrees) of protoDC2 center. star_db_name: str [None] Filename of the database containing stellar sources sed_lookup_dir: str [None] Directory where the SED lookup tables reside. agn_db_name: str [None] Filename of the agn parameter sqlite db file. agn_threads: int [1] Number of threads to use when simulating AGN variability sn_db_name: str [None] Filename of the supernova parameter sqlite db file. sprinkler: bool [False] Flag to enable the Sprinkler. host_image_dir: string The location of the FITS images of lensed AGN/SNe hosts produced by generate_lensed_hosts_***.py host_data_dir: string Location of csv file of lensed host data created by the sprinkler gzip_threads: int The number of gzip jobs that can be started in parallel after catalogs are written (default=3) objects_to_skip: set-like or list-like [()] Collection of object types to skip, e.g., stars, knots, bulges, disks, sne, agn """ self.t_start = time.time() if not os.path.exists(opsimdb): raise RuntimeError('%s does not exist' % opsimdb) self.gzip_threads = gzip_threads # load the data for the parametrized light # curve stellar variability model into a # global cache plc = ParametrizedLightCurveMixin() plc.load_parametrized_light_curves() self.config_dict = config_dict if config_dict is not None else {} self.descqa_catalog = descqa_catalog self.dither = dither self.min_mag = min_mag self.minsource = minsource self.proper_motion = proper_motion self.protoDC2_ra = protoDC2_ra self.protoDC2_dec = protoDC2_dec self.phot_params = PhotometricParameters(nexp=1, exptime=30) self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles() self.obs_gen = ObservationMetaDataGenerator(database=opsimdb, driver='sqlite') if star_db_name is None: raise IOError("Need to specify star_db_name") if not os.path.isfile(star_db_name): raise IOError("%s is not a file\n" % star_db_name + "(This is what you specified for star_db_name") self.star_db = DC2StarObj(database=star_db_name, driver='sqlite') self.sprinkler = sprinkler if self.sprinkler and not HAS_TWINKLES: raise RuntimeError("You are trying to enable the sprinkler; " "but Twinkles cannot be imported") if not os.path.isdir(sed_lookup_dir): raise IOError("\n%s\nis not a dir" % sed_lookup_dir) self.sed_lookup_dir = sed_lookup_dir self._agn_threads = agn_threads if agn_db_name is not None: if os.path.exists(agn_db_name): self.agn_db_name = agn_db_name else: raise IOError("Path to Proto DC2 AGN database does not exist.") else: self.agn_db_name = None self.sn_db_name = None if sn_db_name is not None: if os.path.isfile(sn_db_name): self.sn_db_name = sn_db_name else: raise IOError("%s is not a file" % sn_db_name) if host_image_dir is None and self.sprinkler is not False: raise IOError( "Need to specify the name of the host image directory.") elif self.sprinkler is not False: if os.path.exists(host_image_dir): self.host_image_dir = host_image_dir else: raise IOError("Path to host image directory" + "\n\n%s\n\n" % host_image_dir + "does not exist.") if host_data_dir is None and self.sprinkler is not False: raise IOError( "Need to specify the name of the host data directory.") elif self.sprinkler is not False: if os.path.exists(host_data_dir): self.host_data_dir = host_data_dir else: raise IOError( "Path to host data directory does not exist.\n\n", "%s\n\n" % host_data_dir) self.instcats = get_instance_catalogs() object_types = 'stars knots bulges disks sprinkled hosts sne agn'.split( ) if any([_ not in object_types for _ in objects_to_skip]): raise RuntimeError(f'objects_to_skip ({objects_to_skip}) ' 'contains invalid object types') self.do_obj_type = {_: _ not in objects_to_skip for _ in object_types}
def __init__(self, opsimdb, descqa_catalog, dither=True, min_mag=10, minsource=100, proper_motion=False, imsim_catalog=False, protoDC2_ra=0, protoDC2_dec=0, agn_db_name=None, sprinkler=False): """ Parameters ---------- obsimdb: str OpSim db filename. descqa_catalog: str Name of the DESCQA galaxy catalog. dither: bool [True] Flag to enable the dithering included in the opsim db file. min_mag: float [10] Minimum value of the star magnitude at 500nm to include. minsource: int [100] Minimum number of objects for phosim.py to simulate a chip. proper_motion: bool [True] Flag to enable application of proper motion to stars. imsim_catalog: bool [False] Flag to write an imsim-style object catalog. protoDC2_ra: float [0] Desired RA (J2000 degrees) of protoDC2 center. protoDC2_dec: float [0] Desired Dec (J2000 degrees) of protoDC2 center. agn_db_name: str [None] Filename of the agn parameter sqlite db file. sprinkler: bool [False] Flag to enable the Sprinkler. """ if not os.path.exists(opsimdb): raise RuntimeError('%s does not exist' % opsimdb) # load the data for the parametrized light # curve stellar variability model into a # global cache plc = ParametrizedLightCurveMixin() plc.load_parametrized_light_curves() self.descqa_catalog = descqa_catalog self.dither = dither self.min_mag = min_mag self.minsource = minsource self.proper_motion = proper_motion self.imsim_catalog = imsim_catalog self.protoDC2_ra = protoDC2_ra self.protoDC2_dec = protoDC2_dec self.phot_params = PhotometricParameters(nexp=1, exptime=30) self.bp_dict = BandpassDict.loadTotalBandpassesFromFiles() self.obs_gen = ObservationMetaDataGenerator(database=opsimdb, driver='sqlite') self.star_db = StarObj(database='LSSTCATSIM', host='fatboy.phys.washington.edu', port=1433, driver='mssql+pymssql') if agn_db_name is None: raise IOError("Need to specify an Proto DC2 AGN database.") else: if os.path.exists(agn_db_name): self.agn_db_name = agn_db_name else: raise IOError("Path to Proto DC2 AGN database does not exist.") self.sprinkler = sprinkler self.instcats = get_instance_catalogs(imsim_catalog)
htmid_list = [args.htmid] else: htmid_list = args.htmid else: htmid_list = [] with open(args.htmid_file, 'r') as in_file: for line in in_file: if line.startswith('#'): continue params = line.strip().split() htmid_list.append(int(params[0])) variability_cache = create_variability_cache() plc = ParametrizedLightCurveMixin() plc.load_parametrized_light_curves(variability_cache=variability_cache) variability_cache['rrly_map'] = {} lc_dir = os.path.join(os.environ['SIMS_SED_LIBRARY_DIR'], 'rrly_lc') assert os.path.isdir(lc_dir) with open('data/rrly_lc_map.txt', 'r') as in_file: for line in in_file: p = line.strip().split(';') variability_cache['rrly_map'][int(p[0])] = os.path.join(lc_dir, p[1]) m5_single = {} with open(args.m5_single, 'r') as in_file: for line in in_file: if line.startswith('#'): continue p = line.strip().split()
def test_applyParametrizedLightCurve_manyExpmjd(self): """ test applyParametrizedLightCurve on an array of expmjd values by creating a dummy light curve file with known parameters, generating magnitudes, and comparing to the expected outputs. We will use _calc_dflux() to calculate the known truth, since that method was tested in test_calc_dflux() """ lc_temp_file_name = tempfile.mktemp( prefix='test_applyParametrizedLightCurve_manyexpmjd', suffix='.gz') rng = np.random.RandomState(13291) n_c_1 = 10 a1_list = rng.random_sample(n_c_1) * 5.0 b1_list = (rng.random_sample(n_c_1) - 0.5) * 2.0 c1_list = (rng.random_sample(n_c_1) - 0.5) * 0.1 omega1_list = rng.random_sample(n_c_1) * 20.0 tau1_list = rng.random_sample(n_c_1) * 100.0 median1 = 100.0 n_c_2 = 15 a2_list = rng.random_sample(n_c_2) * 5.0 b2_list = (rng.random_sample(n_c_2) - 0.5) * 2.0 c2_list = (rng.random_sample(n_c_2) - 0.5) * 0.1 omega2_list = rng.random_sample(n_c_2) * 20.0 tau2_list = rng.random_sample(n_c_2) * 100.0 median2 = 200.0 with gzip.open(lc_temp_file_name, 'w') as out_file: out_file.write(b'# a header\n') out_file.write(b'kplr999900000_lc.txt 100 1.0e+02 %d ' % n_c_1) for i_c in range(n_c_1): out_file.write(b'%e ' % (1.0 / (i_c + 1))) out_file.write(b'%e ' % median1) for i_c in range(n_c_1): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a1_list[i_c], b1_list[i_c], c1_list[i_c], omega1_list[i_c], tau1_list[i_c])) out_file.write(b'\n') out_file.write(b'kplr999900001_lc.txt 100 1.0e+02 %d ' % n_c_2) for i_c in range(n_c_2): out_file.write(b'%e ' % (1.0 / (i_c + 1))) out_file.write(b'%e ' % median2) for i_c in range(n_c_2): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a2_list[i_c], b2_list[i_c], c2_list[i_c], omega2_list[i_c], tau2_list[i_c])) out_file.write(b'\n') params = {} params['lc'] = np.array([999900001, 999900000, None, 999900001]) params['t0'] = np.array([223.1, 1781.45, None, 32.0]) kp = ParametrizedLightCurveMixin() kp.load_parametrized_light_curves(lc_temp_file_name) # first test that passing in an empty set of params # results in an empty numpy array (so that the 'dry # run' of catalog generation does not fail) d_mag_out = kp.applyParametrizedLightCurve([], {}, 1.0) np.testing.assert_array_equal(d_mag_out, np.array([[], [], [], [], [], []])) expmjd = rng.random_sample(10) * 10000.0 + 59580.0 d_mag_out = kp.applyParametrizedLightCurve([], params, expmjd) self.assertEqual(d_mag_out.shape, (6, 4, 10)) for i_obj in range(4): if i_obj == 2: for i_filter in range(6): np.testing.assert_array_equal(d_mag_out[i_filter][i_obj], np.zeros(10)) else: q_flux, d_flux = kp._calc_dflux(params['lc'][i_obj], expmjd - params['t0'][i_obj]) d_mag_truth = -2.5 * np.log10(1.0 + d_flux / q_flux) nan_vals = np.where(np.isnan(d_mag_truth)) self.assertEqual(len(nan_vals[0]), 0) for i_filter in range(6): np.testing.assert_array_equal(d_mag_out[i_filter][i_obj], d_mag_truth) sims_clean_up() if os.path.exists(lc_temp_file_name): os.unlink(lc_temp_file_name)
def test_calc_dflux(self): """ Test the method that calculates the flux of parametrized light curves by generating a fake light curve library with known parameters, calculating the fluxes, and comparing to the expected results. """ lc_temp_file_name = tempfile.mktemp(prefix='test_calc_dflux_lc', suffix='.gz') rng = np.random.RandomState(7124) n_c_1 = 10 a1_list = rng.random_sample(n_c_1) * 5.0 b1_list = (rng.random_sample(n_c_1) - 0.5) * 2.0 c1_list = (rng.random_sample(n_c_1) - 0.5) * 0.1 omega1_list = rng.random_sample(n_c_1) * 20.0 tau1_list = rng.random_sample(n_c_1) * 100.0 median1 = 100.0 n_c_2 = 15 a2_list = rng.random_sample(n_c_2) * 5.0 b2_list = (rng.random_sample(n_c_2) - 0.5) * 2.0 c2_list = (rng.random_sample(n_c_2) - 0.5) * 0.1 omega2_list = rng.random_sample(n_c_2) * 20.0 tau2_list = rng.random_sample(n_c_2) * 100.0 median2 = 200.0 with gzip.open(lc_temp_file_name, 'w') as out_file: out_file.write(b'# a header\n') out_file.write(b'kplr990000000_lc.txt 100 1.0e+02 %d ' % n_c_1) for i_c in range(n_c_1): out_file.write(b'%e ' % (1.0 / (i_c + 1))) out_file.write(b'%e ' % median1) for i_c in range(n_c_1): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a1_list[i_c], b1_list[i_c], c1_list[i_c], omega1_list[i_c], tau1_list[i_c])) out_file.write(b'\n') out_file.write(b'kplr990000001_lc.txt 100 1.0e+02 %d ' % n_c_2) for i_c in range(n_c_2): out_file.write(b'%e ' % (1.0 / (i_c + 1))) out_file.write(b'%e ' % median2) for i_c in range(n_c_2): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a2_list[i_c], b2_list[i_c], c2_list[i_c], omega2_list[i_c], tau2_list[i_c])) out_file.write(b'\n') expmjd = rng.random_sample(100) * 200.0 kp = ParametrizedLightCurveMixin() kp.load_parametrized_light_curves(lc_temp_file_name) q_flux, d_flux = kp._calc_dflux(990000000, expmjd) self.assertAlmostEqual(q_flux, median1 + c1_list.sum(), 10) true_flux = np.zeros(len(expmjd)) for i_c in range(n_c_1): arg = omega1_list[i_c] * (expmjd - tau1_list[i_c]) true_flux += a1_list[i_c] * np.cos(arg) true_flux += b1_list[i_c] * np.sin(arg) self.assertEqual(len(d_flux), len(true_flux)) np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10) q_flux, d_flux = kp._calc_dflux(990000001, expmjd) self.assertAlmostEqual(q_flux, median2 + c2_list.sum(), 10) true_flux = np.zeros(len(expmjd)) for i_c in range(n_c_2): arg = omega2_list[i_c] * (expmjd - tau2_list[i_c]) true_flux += a2_list[i_c] * np.cos(arg) true_flux += b2_list[i_c] * np.sin(arg) self.assertEqual(len(d_flux), len(true_flux)) np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10) sims_clean_up() if os.path.exists(lc_temp_file_name): os.unlink(lc_temp_file_name)
def test_applyParametrizedLightCurve_manyExpmjd(self): """ test applyParametrizedLightCurve on an array of expmjd values by creating a dummy light curve file with known parameters, generating magnitudes, and comparing to the expected outputs. We will use _calc_dflux() to calculate the known truth, since that method was tested in test_calc_dflux() """ lc_temp_file_name = tempfile.mktemp(prefix='test_applyParametrizedLightCurve_manyexpmjd', suffix='.gz') rng = np.random.RandomState(13291) n_c_1 = 10 a1_list = rng.random_sample(n_c_1)*5.0 b1_list = (rng.random_sample(n_c_1)-0.5)*2.0 c1_list = (rng.random_sample(n_c_1)-0.5)*0.1 omega1_list = rng.random_sample(n_c_1)*20.0 tau1_list = rng.random_sample(n_c_1)*100.0 median1 = 100.0 n_c_2 = 15 a2_list = rng.random_sample(n_c_2)*5.0 b2_list = (rng.random_sample(n_c_2)-0.5)*2.0 c2_list = (rng.random_sample(n_c_2)-0.5)*0.1 omega2_list = rng.random_sample(n_c_2)*20.0 tau2_list = rng.random_sample(n_c_2)*100.0 median2 = 200.0 with gzip.open(lc_temp_file_name, 'w') as out_file: out_file.write(b'# a header\n') out_file.write(b'kplr999900000_lc.txt 100 1.0e+02 %d ' % n_c_1) for i_c in range(n_c_1): out_file.write(b'%e ' % (1.0/(i_c+1))) out_file.write(b'%e ' % median1) for i_c in range(n_c_1): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a1_list[i_c], b1_list[i_c], c1_list[i_c], omega1_list[i_c], tau1_list[i_c])) out_file.write(b'\n') out_file.write(b'kplr999900001_lc.txt 100 1.0e+02 %d ' % n_c_2) for i_c in range(n_c_2): out_file.write(b'%e ' % (1.0/(i_c+1))) out_file.write(b'%e ' % median2) for i_c in range(n_c_2): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a2_list[i_c], b2_list[i_c], c2_list[i_c], omega2_list[i_c], tau2_list[i_c])) out_file.write(b'\n') params = {} params['lc'] = np.array([999900001, 999900000, None, 999900001]) params['t0'] = np.array([223.1, 1781.45, None, 32.0]) kp = ParametrizedLightCurveMixin() kp.load_parametrized_light_curves(lc_temp_file_name) # first test that passing in an empty set of params # results in an empty numpy array (so that the 'dry # run' of catalog generation does not fail) d_mag_out = kp.applyParametrizedLightCurve([],{},1.0) np.testing.assert_array_equal(d_mag_out, np.array([[],[],[],[],[],[]])) expmjd = rng.random_sample(10)*10000.0 + 59580.0 d_mag_out = kp.applyParametrizedLightCurve([], params, expmjd) self.assertEqual(d_mag_out.shape, (6, 4, 10)) for i_obj in range(4): if i_obj == 2: for i_filter in range(6): np.testing.assert_array_equal(d_mag_out[i_filter][i_obj], np.zeros(10)) else: q_flux, d_flux = kp._calc_dflux(params['lc'][i_obj], expmjd-params['t0'][i_obj]) d_mag_truth = -2.5*np.log10(1.0+d_flux/q_flux) nan_vals = np.where(np.isnan(d_mag_truth)) self.assertEqual(len(nan_vals[0]), 0) for i_filter in range(6): np.testing.assert_array_equal(d_mag_out[i_filter][i_obj], d_mag_truth) sims_clean_up() if os.path.exists(lc_temp_file_name): os.unlink(lc_temp_file_name)
def test_calc_dflux(self): """ Test the method that calculates the flux of parametrized light curves by generating a fake light curve library with known parameters, calculating the fluxes, and comparing to the expected results. """ lc_temp_file_name = tempfile.mktemp(prefix='test_calc_dflux_lc', suffix='.gz') rng = np.random.RandomState(7124) n_c_1 = 10 a1_list = rng.random_sample(n_c_1)*5.0 b1_list = (rng.random_sample(n_c_1)-0.5)*2.0 c1_list = (rng.random_sample(n_c_1)-0.5)*0.1 omega1_list = rng.random_sample(n_c_1)*20.0 tau1_list = rng.random_sample(n_c_1)*100.0 median1 = 100.0 n_c_2 = 15 a2_list = rng.random_sample(n_c_2)*5.0 b2_list = (rng.random_sample(n_c_2)-0.5)*2.0 c2_list = (rng.random_sample(n_c_2)-0.5)*0.1 omega2_list = rng.random_sample(n_c_2)*20.0 tau2_list = rng.random_sample(n_c_2)*100.0 median2 = 200.0 with gzip.open(lc_temp_file_name, 'w') as out_file: out_file.write(b'# a header\n') out_file.write(b'kplr990000000_lc.txt 100 1.0e+02 %d ' % n_c_1) for i_c in range(n_c_1): out_file.write(b'%e ' % (1.0/(i_c+1))) out_file.write(b'%e ' % median1) for i_c in range(n_c_1): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a1_list[i_c], b1_list[i_c], c1_list[i_c], omega1_list[i_c], tau1_list[i_c])) out_file.write(b'\n') out_file.write(b'kplr990000001_lc.txt 100 1.0e+02 %d ' % n_c_2) for i_c in range(n_c_2): out_file.write(b'%e ' % (1.0/(i_c+1))) out_file.write(b'%e ' % median2) for i_c in range(n_c_2): out_file.write(b'%.15e %.15e %.15e %.15e %.15e ' % (a2_list[i_c], b2_list[i_c], c2_list[i_c], omega2_list[i_c], tau2_list[i_c])) out_file.write(b'\n') expmjd = rng.random_sample(100)*200.0 kp = ParametrizedLightCurveMixin() kp.load_parametrized_light_curves(lc_temp_file_name) q_flux, d_flux = kp._calc_dflux(990000000, expmjd) self.assertAlmostEqual(q_flux, median1+c1_list.sum(), 10) true_flux = np.zeros(len(expmjd)) for i_c in range(n_c_1): arg = omega1_list[i_c]*(expmjd-tau1_list[i_c]) true_flux += a1_list[i_c]*np.cos(arg) true_flux += b1_list[i_c]*np.sin(arg) self.assertEqual(len(d_flux), len(true_flux)) np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10) q_flux, d_flux = kp._calc_dflux(990000001, expmjd) self.assertAlmostEqual(q_flux, median2+c2_list.sum(), 10) true_flux = np.zeros(len(expmjd)) for i_c in range(n_c_2): arg = omega2_list[i_c]*(expmjd-tau2_list[i_c]) true_flux += a2_list[i_c]*np.cos(arg) true_flux += b2_list[i_c]*np.sin(arg) self.assertEqual(len(d_flux), len(true_flux)) np.testing.assert_allclose(d_flux, true_flux, rtol=0.0, atol=1.0e-10) sims_clean_up() if os.path.exists(lc_temp_file_name): os.unlink(lc_temp_file_name)