def test_mags_vs_flux(self): """ Verify that the relationship between Sed.calcMag() and Sed.calcFlux() is as expected """ wavelen = np.arange(100.0, 1500.0, 1.0) flambda = np.exp(-0.5*np.power((wavelen-500.0)/100.0,2)) sb = (wavelen-100.0)/1400.0 ss = Sed(wavelen=wavelen, flambda=flambda) bp = Bandpass(wavelen=wavelen, sb=sb) mag = ss.calcMag(bp) flux = ss.calcFlux(bp) self.assertAlmostEqual(ss.magFromFlux(flux)/mag, 1.0, 10) self.assertAlmostEqual(ss.fluxFromMag(mag)/flux, 1.0, 10)
def test_mags_vs_flux(self): """ Verify that the relationship between Sed.calcMag() and Sed.calcFlux() is as expected """ wavelen = np.arange(100.0, 1500.0, 1.0) flambda = np.exp(-0.5 * np.power((wavelen - 500.0) / 100.0, 2)) sb = (wavelen - 100.0) / 1400.0 ss = Sed(wavelen=wavelen, flambda=flambda) bp = Bandpass(wavelen=wavelen, sb=sb) mag = ss.calcMag(bp) flux = ss.calcFlux(bp) self.assertAlmostEqual(ss.magFromFlux(flux) / mag, 1.0, 10) self.assertAlmostEqual(ss.fluxFromMag(mag) / flux, 1.0, 10)
def calculate_mags(galaxy_list, out_dict): """ Calculate the total (bulge+disk+agn) magnitudes of galaxies Parameters ---------- galaxy_list is a list of tuples. The tuples are of the form (bulge.sed_name, bulge.magNorm, disk.sed_name, disk.magNorm agn.sed_name, agn.magNorm) out_dict is a Multiprocessing.Manager.dict object that will store the results of this calculation """ i_process = mp.current_process().pid bulge_fluxes = np.zeros((len(galaxy_list), 6), dtype=float) disk_fluxes = np.zeros((len(galaxy_list), 6), dtype=float) magnification = np.array( [1.0 / ((1.0 - g[14])**2 - g[12]**2 - g[13]**2) for g in galaxy_list]) assert magnification.min() >= 0.0 assert len(np.where(np.isfinite(magnification))[0]) == len(magnification) for i_gal, galaxy in enumerate(galaxy_list): if galaxy[0] is not None and galaxy[1] is not None: bulge_fluxes[i_gal] = _fluxes(galaxy[0], galaxy[1], galaxy[6]) if galaxy[2] is not None and galaxy[3] is not None: disk_fluxes[i_gal] = _fluxes(galaxy[2], galaxy[3], galaxy[6]) tot_fluxes = bulge_fluxes + disk_fluxes for i_filter in range(6): tot_fluxes[:, i_filter] *= magnification dummy_sed = Sed() valid = np.where(tot_fluxes > 0.0) valid_mags = dummy_sed.magFromFlux(tot_fluxes[valid]) out_mags = np.NaN * np.ones((len(galaxy_list), 6), dtype=float) out_mags[valid] = valid_mags out_dict[i_process] = out_mags
def test_alert_data_generation(self): dmag_cutoff = 0.005 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5} _max_var_param_str = self.max_str_len class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject): objid = 'star_alert' tableid = 'stars' idColKey = 'simobjid' raColName = 'ra' decColName = 'dec' objectTypeId = 0 columns = [('raJ2000', 'ra*0.01745329252'), ('decJ2000', 'dec*0.01745329252'), ('parallax', 'px*0.01745329252/3600.0'), ('properMotionRa', 'pmra*0.01745329252/3600.0'), ('properMotionDec', 'pmdec*0.01745329252/3600.0'), ('radialVelocity', 'vrad'), ('variabilityParameters', 'varParamStr', str, _max_var_param_str)] class TestAlertsVarCatMixin(object): @register_method('alert_test') def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None): if len(params) == 0: return np.array([[], [], [], [], [], []]) if isinstance(expmjd, numbers.Number): dmags_out = np.zeros((6, self.num_variable_obj(params))) else: dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd))) for i_star in range(self.num_variable_obj(params)): if params['amp'][i_star] is not None: dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd) for i_filter in range(6): dmags_out[i_filter][i_star] = dmags return dmags_out class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog): pass class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoords, AstrometryStars, Variability, InstanceCatalog): column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert'] camera = obs_lsst_phosim.PhosimMapper().camera @compound('delta_umag', 'delta_gmag', 'delta_rmag', 'delta_imag', 'delta_zmag', 'delta_ymag') def get_TruthVariability(self): return self.applyVariability(self.column_by_name('varParamStr')) @cached def get_dmagAlert(self): return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass) @cached def get_magAlert(self): return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \ self.column_by_name('dmagAlert') star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite') # assemble the true light curves for each object; we need to figure out # if their np.max(dMag) ever goes over dmag_cutoff; then we will know if # we are supposed to simulate them true_lc_dict = {} true_lc_obshistid_dict = {} is_visible_dict = {} obs_dict = {} max_obshistid = -1 n_total_observations = 0 for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] if obshistid > max_obshistid: max_obshistid = obshistid cat = TestAlertsTruthCat(star_db, obs_metadata=obs) for line in cat.iter_catalog(): if line[1] is None: continue n_total_observations += 1 if line[0] not in true_lc_dict: true_lc_dict[line[0]] = {} true_lc_obshistid_dict[line[0]] = [] true_lc_dict[line[0]][obshistid] = line[2] true_lc_obshistid_dict[line[0]].append(obshistid) if line[0] not in is_visible_dict: is_visible_dict[line[0]] = False if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]: is_visible_dict[line[0]] = True obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2))) skipped_due_to_mag = 0 objects_to_simulate = [] obshistid_unqid_set = set() for obj_id in true_lc_dict: dmag_max = -1.0 for obshistid in true_lc_dict[obj_id]: if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max: dmag_max = np.abs(true_lc_dict[obj_id][obshistid]) if dmag_max >= dmag_cutoff: if not is_visible_dict[obj_id]: skipped_due_to_mag += 1 continue objects_to_simulate.append(obj_id) for obshistid in true_lc_obshistid_dict[obj_id]: obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid) self.assertGreater(len(objects_to_simulate), 10) self.assertGreater(skipped_due_to_mag, 0) log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid(htmid, star_db, photometry_class=TestAlertsVarCat, output_prefix='alert_test', output_dir=self.output_dir, dmag_cutoff=dmag_cutoff, log_file_name=log_file_name) dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() phot_params = PhotometricParameters() # First, verify that the contents of the sqlite files are all correct n_tot_simulated = 0 alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, ' alert_query += 'meta.band, quiescent.flux, alert.dflux, ' alert_query += 'quiescent.snr, alert.snr, ' alert_query += 'alert.ra, alert.dec, alert.chipNum, ' alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, ' alert_query += 'ast.parallax ' alert_query += 'FROM alert_data AS alert ' alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId ' alert_query += 'INNER JOIN quiescent_flux AS quiescent ' alert_query += 'ON quiescent.uniqueId=alert.uniqueId ' alert_query += 'AND quiescent.band=meta.band ' alert_query += 'INNER JOIN baseline_astrometry AS ast ' alert_query += 'ON ast.uniqueId=alert.uniqueId' alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int), ('TAI', float), ('band', int), ('q_flux', float), ('dflux', float), ('q_snr', float), ('tot_snr', float), ('ra', float), ('dec', float), ('chipNum', int), ('xPix', float), ('yPix', float), ('pmRA', float), ('pmDec', float), ('parallax', float)]) sqlite_file_list = os.listdir(self.output_dir) n_tot_simulated = 0 obshistid_unqid_simulated_set = set() for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype) if len(alert_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI']) for i_obj in range(len(alert_data)): n_tot_simulated += 1 obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) + alert_data['obshistId'][i_obj]) unq = alert_data['uniqueId'][i_obj] obj_dex = (unq//1024)-1 self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4) self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4) self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4) ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) distance_arcsec = 3600.0*distance msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) self.assertLess(distance_arcsec, 0.0005, msg=msg) obs = obs_dict[alert_data['obshistId'][i_obj]] chipname = chipNameFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, camera=self.camera) chipnum = int(chipname.replace('R', '').replace('S', ''). replace(' ', '').replace(';', '').replace(',', ''). replace(':', '')) self.assertEqual(chipnum, alert_data['chipNum'][i_obj]) xpix, ypix = pixelCoordsFromRaDec(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, camera=self.camera) self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4) self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4) dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj]) self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]], dmag_sim, 3) mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]] m5 = obs.m5[mag_name] q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj]) self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], q_mag, 4) snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], bp_dict[mag_name], self.obs_mag_cutoff[alert_data['band'][i_obj]], phot_params) self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4) tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \ true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]] snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name], m5, phot_params) self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4) for val in obshistid_unqid_set: self.assertIn(val, obshistid_unqid_simulated_set) self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set)) astrometry_query = 'SELECT uniqueId, ra, dec, TAI ' astrometry_query += 'FROM baseline_astrometry' astrometry_dtype = np.dtype([('uniqueId', int), ('ra', float), ('dec', float), ('TAI', float)]) tai_list = [] for obs in self.obs_list: tai_list.append(obs.mjd.TAI) tai_list = np.array(tai_list) n_tot_ast_simulated = 0 for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype) if len(astrometry_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI']) for i_obj in range(len(astrometry_data)): n_tot_ast_simulated += 1 obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1 ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, astrometry_data['ra'][i_obj], astrometry_data['dec'][i_obj]) self.assertLess(3600.0*distance, 0.0005) del alert_gen gc.collect() self.assertGreater(n_tot_simulated, 10) self.assertGreater(len(obshistid_unqid_simulated_set), 10) self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations) self.assertGreater(n_tot_ast_simulated, 0)
def applyMLTflaring(self, valid_dexes, params, expmjd, parallax=None, ebv=None, quiescent_mags=None): """ parallax, ebv, and quiescent_mags are optional kwargs for use if you are calling this method outside the context of an InstanceCatalog (presumably with a numpy array of expmjd) parallax is the parallax of your objects in radians ebv is the E(B-V) value for your objects quiescent_mags is a dict keyed on ('u', 'g', 'r', 'i', 'z', 'y') with the quiescent magnitudes of the objects """ if parallax is None: parallax = self.column_by_name('parallax') if ebv is None: ebv = self.column_by_name('ebv') global _MLT_LC_NPZ global _MLT_LC_NPZ_NAME global _MLT_LC_TIME_CACHE global _MLT_LC_FLUX_CACHE # this needs to occur before loading the MLT light curve cache, # just in case the user wants to override the light curve cache # file by hand before generating the catalog if len(params) == 0: return numpy.array([[],[],[],[],[],[]]) if quiescent_mags is None: quiescent_mags = {} for mag_name in ('u', 'g', 'r', 'i', 'z', 'y'): if ('lsst_%s' % mag_name in self._actually_calculated_columns or 'delta_lsst_%s' % mag_name in self._actually_calculated_columns): quiescent_mags[mag_name] = self.column_by_name('quiescent_lsst_%s' % mag_name) if not hasattr(self, 'photParams'): raise RuntimeError("To apply MLT dwarf flaring, your " "InstanceCatalog must have a member variable " "photParams which is an instantiation of the " "class PhotometricParameters, which can be " "imported from lsst.sims.photUtils. " "This is so that your InstanceCatalog has " "knowledge of the effective area of the LSST " "mirror.") if _MLT_LC_NPZ is None or _MLT_LC_NPZ_NAME != self._mlt_lc_file or _MLT_LC_NPZ.fid is None: if not os.path.exists(self._mlt_lc_file): catutils_scripts = os.path.join(getPackageDir('sims_catUtils'), 'support_scripts') raise RuntimeError("The MLT flaring light curve file:\n" + "\n%s\n" % self._mlt_lc_file + "\ndoes not exist." +"\n\n" + "Go into %s " % catutils_scripts + "and run get_mdwarf_flares.sh " + "to get the data") _MLT_LC_NPZ = numpy.load(self._mlt_lc_file) sims_clean_up.targets.append(_MLT_LC_NPZ) _MLT_LC_NPZ_NAME = self._mlt_lc_file _MLT_LC_TIME_CACHE = {} _MLT_LC_FLUX_CACHE = {} if not hasattr(self, '_mlt_dust_lookup'): # Construct a look-up table to determine the factor # by which to multiply the flares' flux to account for # dust as a function of E(B-V). Recall that we are # modeling all MLT flares as 9000K blackbodies. if not hasattr(self, 'lsstBandpassDict'): raise RuntimeError('You are asking for MLT dwarf flaring ' 'magnitudes in a catalog that has not ' 'defined lsstBandpassDict. The MLT ' 'flaring magnitudes model does not know ' 'how to apply dust extinction to the ' 'flares without the member variable ' 'lsstBandpassDict being defined.') ebv_grid = numpy.arange(0.0, 7.01, 0.01) bb_wavelen = numpy.arange(200.0, 1500.0, 0.1) hc_over_k = 1.4387e7 # nm*K temp = 9000.0 # black body temperature in Kelvin exp_arg = hc_over_k/(temp*bb_wavelen) exp_term = 1.0/(numpy.exp(exp_arg) - 1.0) ln_exp_term = numpy.log(exp_term) # Blackbody f_lambda function; # discard normalizing factors; we only care about finding the # ratio of fluxes between the case with dust extinction and # the case without dust extinction log_bb_flambda = -5.0*numpy.log(bb_wavelen) + ln_exp_term bb_flambda = numpy.exp(log_bb_flambda) bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda) base_fluxes = self.lsstBandpassDict.fluxListForSed(bb_sed) a_x, b_x = bb_sed.setupCCMab() self._mlt_dust_lookup = {} self._mlt_dust_lookup['ebv'] = ebv_grid list_of_bp = self.lsstBandpassDict.keys() for bp in list_of_bp: self._mlt_dust_lookup[bp] = numpy.zeros(len(ebv_grid)) for iebv, ebv_val in enumerate(ebv_grid): wv, fl = bb_sed.addCCMDust(a_x, b_x, ebv=ebv_val, wavelen=bb_wavelen, flambda=bb_flambda) dusty_bb = Sed(wavelen=wv, flambda=fl) dusty_fluxes = self.lsstBandpassDict.fluxListForSed(dusty_bb) for ibp, bp in enumerate(list_of_bp): self._mlt_dust_lookup[bp][iebv] = dusty_fluxes[ibp]/base_fluxes[ibp] # get the distance to each star in parsecs _au_to_parsec = 1.0/206265.0 dd = _au_to_parsec/parallax # get the area of the sphere through which the star's energy # is radiating to get to us (in cm^2) _cm_per_parsec = 3.08576e16 sphere_area = 4.0*numpy.pi*numpy.power(dd*_cm_per_parsec, 2) flux_factor = self.photParams.effarea/sphere_area if isinstance(expmjd, numbers.Number): dMags = numpy.zeros((6, self.num_variable_obj(params))) else: dMags = numpy.zeros((6, self.num_variable_obj(params), len(expmjd))) mag_name_tuple = ('u', 'g', 'r', 'i', 'z', 'y') base_fluxes = {} base_mags = {} ss = Sed() for mag_name in mag_name_tuple: if ('lsst_%s' % mag_name in self._actually_calculated_columns or 'delta_lsst_%s' % mag_name in self._actually_calculated_columns): mm = quiescent_mags[mag_name] base_mags[mag_name] = mm base_fluxes[mag_name] = ss.fluxFromMag(mm) lc_name_arr = params['lc'].astype(str) lc_names_unique = numpy.unique(lc_name_arr) for lc_name in lc_names_unique: if 'None' in lc_name: continue use_this_lc = numpy.where(numpy.char.find(lc_name_arr, lc_name)==0) lc_name = lc_name.replace('.txt', '') # 2017 May 1 # There isn't supposed to be a 'late_inactive' light curve. # Unfortunately, I (Scott Daniel) assigned 'late_inactive' # light curves to some of the stars on our database. Rather # than fix the database table (which will take about a week of # compute time), I am going to fix the problem here by mapping # 'late_inactive' into 'late_active'. if 'late' in lc_name: lc_name = lc_name.replace('in', '') if lc_name in _MLT_LC_TIME_CACHE: raw_time_arr = _MLT_LC_TIME_CACHE[lc_name] else: raw_time_arr = _MLT_LC_NPZ['%s_time' % lc_name] _MLT_LC_TIME_CACHE[lc_name] = raw_time_arr time_arr = self._survey_start + raw_time_arr dt = time_arr.max() - time_arr.min() if isinstance(expmjd, numbers.Number): t_interp = (expmjd + params['t0'][use_this_lc]).astype(float) else: n_obj = len(use_this_lc[0]) n_time = len(expmjd) t_interp = numpy.ones(shape=(n_obj, n_time))*expmjd t_interp += numpy.array([[tt]*n_time for tt in params['t0'][use_this_lc].astype(float)]) while t_interp.max() > time_arr.max(): bad_dexes = numpy.where(t_interp>time_arr.max()) t_interp[bad_dexes] -= dt for i_mag, mag_name in enumerate(mag_name_tuple): if ('lsst_%s' % mag_name in self._actually_calculated_columns or 'delta_lsst_%s' % mag_name in self._actually_calculated_columns): flux_name = '%s_%s' % (lc_name, mag_name) if flux_name in _MLT_LC_FLUX_CACHE: flux_arr = _MLT_LC_FLUX_CACHE[flux_name] else: flux_arr = _MLT_LC_NPZ[flux_name] _MLT_LC_FLUX_CACHE[flux_name] = flux_arr dflux = numpy.interp(t_interp, time_arr, flux_arr) if isinstance(expmjd, numbers.Number): dflux *= flux_factor[use_this_lc] else: dflux *= numpy.array([flux_factor[use_this_lc]]*n_time).transpose() dust_factor = numpy.interp(ebv[use_this_lc], self._mlt_dust_lookup['ebv'], self._mlt_dust_lookup[mag_name]) if not isinstance(expmjd, numbers.Number): dust_factor = numpy.array([dust_factor]*n_time).transpose() dflux *= dust_factor if isinstance(expmjd, numbers.Number): local_base_fluxes = base_fluxes[mag_name][use_this_lc] local_base_mags = base_mags[mag_name][use_this_lc] else: local_base_fluxes = numpy.array([base_fluxes[mag_name][use_this_lc]]*n_time).transpose() local_base_mags = numpy.array([base_mags[mag_name][use_this_lc]]*n_time).transpose() dMags[i_mag][use_this_lc] = (ss.magFromFlux(local_base_fluxes + dflux) - local_base_mags) return dMags
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, m5_single, obs_md_list, proper_chip, variability_cache, out_data, lock): t_start_chunk = time.time() #print('processing %d' % len(chunk)) ct_first = 0 ct_at_all = 0 ct_tot = 0 n_t = len(filter_obs) n_obj = len(chunk) coadd_visits = {} coadd_visits['u'] = 6 coadd_visits['g'] = 8 coadd_visits['r'] = 18 coadd_visits['i'] = 18 coadd_visits['z'] = 16 coadd_visits['y'] = 16 gamma_coadd = {} for bp in 'ugrizy': gamma_coadd[bp] = None gamma_single = {} for bp in 'ugrizy': gamma_single[bp] = [None]*n_t dflux = np.zeros((n_obj,n_t), dtype=float) dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux) dummy_sed = Sed() lsst_bp = BandpassDict.loadTotalBandpassesFromFiles() flux_q = np.zeros((6,n_obj), dtype=float) flux_coadd = np.zeros((6,n_obj), dtype=float) mag_coadd = np.zeros((6,n_obj), dtype=float) snr_coadd = np.zeros((6,n_obj), dtype=float) snr_single = {} snr_single_mag_grid = np.arange(14.0, 30.0, 0.05) phot_params_single = PhotometricParameters(nexp=1, exptime=30.0) t_start_snr = time.time() photometry_mask = np.zeros((n_obj, n_t), dtype=bool) photometry_mask_1d = np.zeros(n_obj, dtype=bool) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) phot_params_coadd = PhotometricParameters(nexp=1, exptime=30.0*coadd_visits[bp]) flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp]) dflux_sub = dflux[:,valid_obs[0]] assert dflux_sub.shape == (n_obj, len(valid_obs[0])) dflux_mean = np.mean(dflux_sub, axis=1) assert dflux_mean.shape==(n_obj,) flux_coadd[i_bp] = flux_q[i_bp]+dflux_mean mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp]) (snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp], coadd_m5[bp], phot_params_coadd) (snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid, lsst_bp[bp], m5_single[bp], phot_params_single) #print('got all snr in %e' % (time.time()-t_start_snr)) t_start_obj = time.time() snr_arr = np.zeros((n_obj, n_t), dtype=float) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs==i_bp) if len(valid_obs[0])==0: continue n_bp = len(valid_obs[0]) dflux_bp = dflux[:,valid_obs[0]] flux0_arr = flux_q[i_bp] flux_tot = flux0_arr[:,None] + dflux_bp mag_tot = dummy_sed.magFromFlux(flux_tot) snr_single_val = np.interp(mag_tot, snr_single_mag_grid, snr_single[bp]) noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp] noise_single = flux_tot/snr_single_val noise = np.sqrt(noise_coadd[:,None]**2+noise_single**2) dflux_thresh = 5.0*noise dflux_bp = np.abs(dflux_bp) detected = (dflux_bp>=dflux_thresh) snr_arr[:,valid_obs[0]] = dflux_bp/noise for i_obj in range(n_obj): if detected[i_obj].any(): photometry_mask_1d[i_obj] = True photometry_mask[i_obj,valid_obs[0]] = detected[i_obj] t_before_chip = time.time() chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'], photometry_mask_1d, obs_md_list, filter_obs, proper_chip) duration = (time.time()-t_before_chip)/3600.0 unq_out = -1*np.ones(n_obj, dtype=int) mjd_out = -1.0*np.ones(n_obj, dtype=float) snr_out = -1.0*np.ones(n_obj, dtype=float) var_type_out = -1*np.ones(n_obj, dtype=int) for i_obj in range(n_obj): if photometry_mask_1d[i_obj]: detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:] if detected.any(): unq_out[i_obj] = chunk['simobjid'][i_obj] first_dex = np.where(detected)[0].min() mjd_out[i_obj] = mjd_obs[first_dex] snr_out[i_obj] = snr_arr[i_obj, first_dex] var_type_out[i_obj] = chunk['var_type'][i_obj] valid = np.where(unq_out>=0) unq_out = unq_out[valid] mjd_out = mjd_out[valid] var_type_out = var_type_out[valid] snr_out = snr_out[valid] with lock: existing_keys = list(out_data.keys()) key_val = 0 if len(existing_keys)>0: key_val = max(existing_keys) while key_val in existing_keys: key_val += 1 out_data[key_val] = (None, None, None, None) out_data[key_val] = (unq_out, mjd_out, snr_out, var_type_out)
def test_flare_magnitudes_mixed_with_dummy(self): """ Test that we get the expected magnitudes out """ db = MLT_test_DB(database=self.db_name, driver='sqlite') # load the quiescent SEDs of the objects in our catalog sed_list = SedList(['lte028-5.0+0.5a+0.0.BT-Settl.spec.gz']*4, [17.1, 17.2, 17.3, 17.4], galacticAvList = [2.432, 1.876, 2.654, 2.364], fileDir=getPackageDir('sims_sed_library'), specMap=defaultSpecMap) bp_dict = BandpassDict.loadTotalBandpassesFromFiles() # calculate the quiescent fluxes of the objects in our catalog baseline_fluxes = bp_dict.fluxListForSedList(sed_list) bb_wavelen = np.arange(100.0, 1600.0, 0.1) bb_flambda = blackbody_lambda(bb_wavelen*10.0, 9000.0) # this data is taken from the setUpClass() classmethod above t0_list = [456.2, 41006.2, 117.2, 10456.2] av_list = [2.432, 1.876, 2.654, 2.364] parallax_list = np.array([0.25, 0.15, 0.3, 0.22]) distance_list = 1.0/(206265.0*radiansFromArcsec(0.001*parallax_list)) distance_list *= 3.0857e18 # convert to cm dtype = np.dtype([('id', int), ('u', float), ('g', float)]) photParams = PhotometricParameters() ss = Sed() quiet_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_dummy_quiet_cat.txt') flare_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_dummy_flaring_cat.txt') # loop over several MJDs and verify that, to within a # milli-mag, our flaring model gives us the magnitudes # expected, given the light curves specified in # setUpClass() for mjd in (59580.0, 60000.0, 70000.0, 80000.0): obs = ObservationMetaData(mjd=mjd) quiet_cat = QuiescentCatalog(db, obs_metadata=obs) quiet_cat.write_catalog(quiet_cat_name) flare_cat = FlaringCatalogDummy(db, obs_metadata=obs) flare_cat.scratch_dir = self.scratch_dir flare_cat._mlt_lc_file = self.mlt_lc_name flare_cat.write_catalog(flare_cat_name) quiescent_data = np.genfromtxt(quiet_cat_name, dtype=dtype, delimiter=',') flaring_data = np.genfromtxt(flare_cat_name, dtype=dtype, delimiter=',') self.assertGreater(len(quiescent_data), 2) self.assertEqual(len(quiescent_data), len(flaring_data)) self.assertIn(3, flaring_data['id']) for ix in range(len(flaring_data)): obj_id = flaring_data['id'][ix] self.assertEqual(obj_id, ix) msg = ('failed on object %d; mjd %.2f\n u_quiet %e u_flare %e\n g_quiet %e g_flare %e' % (obj_id, mjd, quiescent_data['u'][obj_id], flaring_data['u'][obj_id], quiescent_data['g'][obj_id], flaring_data['g'][obj_id])) self.assertEqual(quiescent_data['id'][obj_id], flaring_data['id'][obj_id], msg=msg) self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][0]), quiescent_data['u'][obj_id], 3, msg=msg) self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][1]), quiescent_data['g'][obj_id], 3, msg=msg) if obj_id != 3: # the models below are as specified in the # setUpClass() method if obj_id == 0 or obj_id == 1: amp = 1.0e42 dt = 3652.5 t_min = flare_cat._survey_start - t0_list[obj_id] tt = mjd - t_min while tt > dt: tt -= dt u_flux = amp*(1.0+np.power(np.sin(tt/100.0), 2)) g_flux = amp*(1.0+np.power(np.cos(tt/100.0), 2)) elif obj_id==2: amp = 2.0e41 dt = 365.25 t_min = flare_cat._survey_start - t0_list[obj_id] tt = mjd - t_min while tt > dt: tt -= dt u_flux = amp*(1.0+np.power(np.sin(tt/50.0), 2)) g_flux = amp*(1.0+np.power(np.cos(tt/50.0), 2)) # calculate the multiplicative effect of dust on a 9000K # black body bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda) u_bb_flux = bb_sed.calcFlux(bp_dict['u']) g_bb_flux = bb_sed.calcFlux(bp_dict['g']) a_x, b_x = bb_sed.setupCCM_ab() bb_sed.addDust(a_x, b_x, A_v=av_list[obj_id]) u_bb_dusty_flux = bb_sed.calcFlux(bp_dict['u']) g_bb_dusty_flux = bb_sed.calcFlux(bp_dict['g']) dust_u = u_bb_dusty_flux/u_bb_flux dust_g = g_bb_dusty_flux/g_bb_flux area = 4.0*np.pi*np.power(distance_list[obj_id], 2) tot_u_flux = baseline_fluxes[obj_id][0] + u_flux*dust_u/area tot_g_flux = baseline_fluxes[obj_id][1] + g_flux*dust_g/area self.assertAlmostEqual(ss.magFromFlux(tot_u_flux), flaring_data['u'][obj_id], 3, msg=msg) self.assertAlmostEqual(ss.magFromFlux(tot_g_flux), flaring_data['g'][obj_id], 3, msg=msg) self.assertGreater(np.abs(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id]), 0.001, msg=msg) self.assertGreater(np.abs(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id]), 0.001, msg=msg) else: self.assertAlmostEqual(flaring_data['g'][obj_id], quiescent_data['g'][obj_id]+3*(mjd-59580.0)/10000.0, 3, msg=msg) self.assertAlmostEqual(flaring_data['u'][obj_id], quiescent_data['u'][obj_id]+2*(mjd-59580.0)/10000.0, 3, msg=msg) if os.path.exists(quiet_cat_name): os.unlink(quiet_cat_name) if os.path.exists(flare_cat_name): os.unlink(flare_cat_name)
def test_alert_data_generation(self): dmag_cutoff = 0.005 mag_name_to_int = {'u': 0, 'g': 1, 'r': 2, 'i': 3, 'z' : 4, 'y': 5} _max_var_param_str = self.max_str_len class StarAlertTestDBObj(StellarAlertDBObjMixin, CatalogDBObject): objid = 'star_alert' tableid = 'stars' idColKey = 'simobjid' raColName = 'ra' decColName = 'dec' objectTypeId = 0 columns = [('raJ2000', 'ra*0.01745329252'), ('decJ2000', 'dec*0.01745329252'), ('parallax', 'px*0.01745329252/3600.0'), ('properMotionRa', 'pmra*0.01745329252/3600.0'), ('properMotionDec', 'pmdec*0.01745329252/3600.0'), ('radialVelocity', 'vrad'), ('variabilityParameters', 'varParamStr', str, _max_var_param_str)] class TestAlertsVarCatMixin(object): @register_method('alert_test') def applyAlertTest(self, valid_dexes, params, expmjd, variability_cache=None): if len(params) == 0: return np.array([[], [], [], [], [], []]) if isinstance(expmjd, numbers.Number): dmags_out = np.zeros((6, self.num_variable_obj(params))) else: dmags_out = np.zeros((6, self.num_variable_obj(params), len(expmjd))) for i_star in range(self.num_variable_obj(params)): if params['amp'][i_star] is not None: dmags = params['amp'][i_star]*np.cos(params['per'][i_star]*expmjd) for i_filter in range(6): dmags_out[i_filter][i_star] = dmags return dmags_out class TestAlertsVarCat(TestAlertsVarCatMixin, AlertStellarVariabilityCatalog): pass class TestAlertsTruthCat(TestAlertsVarCatMixin, CameraCoordsLSST, AstrometryStars, Variability, InstanceCatalog): column_outputs = ['uniqueId', 'chipName', 'dmagAlert', 'magAlert'] @compound('delta_umag', 'delta_gmag', 'delta_rmag', 'delta_imag', 'delta_zmag', 'delta_ymag') def get_TruthVariability(self): return self.applyVariability(self.column_by_name('varParamStr')) @cached def get_dmagAlert(self): return self.column_by_name('delta_%smag' % self.obs_metadata.bandpass) @cached def get_magAlert(self): return self.column_by_name('%smag' % self.obs_metadata.bandpass) + \ self.column_by_name('dmagAlert') star_db = StarAlertTestDBObj(database=self.star_db_name, driver='sqlite') # assemble the true light curves for each object; we need to figure out # if their np.max(dMag) ever goes over dmag_cutoff; then we will know if # we are supposed to simulate them true_lc_dict = {} true_lc_obshistid_dict = {} is_visible_dict = {} obs_dict = {} max_obshistid = -1 n_total_observations = 0 for obs in self.obs_list: obs_dict[obs.OpsimMetaData['obsHistID']] = obs obshistid = obs.OpsimMetaData['obsHistID'] if obshistid > max_obshistid: max_obshistid = obshistid cat = TestAlertsTruthCat(star_db, obs_metadata=obs) for line in cat.iter_catalog(): if line[1] is None: continue n_total_observations += 1 if line[0] not in true_lc_dict: true_lc_dict[line[0]] = {} true_lc_obshistid_dict[line[0]] = [] true_lc_dict[line[0]][obshistid] = line[2] true_lc_obshistid_dict[line[0]].append(obshistid) if line[0] not in is_visible_dict: is_visible_dict[line[0]] = False if line[3] <= self.obs_mag_cutoff[mag_name_to_int[obs.bandpass]]: is_visible_dict[line[0]] = True obshistid_bits = int(np.ceil(np.log(max_obshistid)/np.log(2))) skipped_due_to_mag = 0 objects_to_simulate = [] obshistid_unqid_set = set() for obj_id in true_lc_dict: dmag_max = -1.0 for obshistid in true_lc_dict[obj_id]: if np.abs(true_lc_dict[obj_id][obshistid]) > dmag_max: dmag_max = np.abs(true_lc_dict[obj_id][obshistid]) if dmag_max >= dmag_cutoff: if not is_visible_dict[obj_id]: skipped_due_to_mag += 1 continue objects_to_simulate.append(obj_id) for obshistid in true_lc_obshistid_dict[obj_id]: obshistid_unqid_set.add((obj_id << obshistid_bits) + obshistid) self.assertGreater(len(objects_to_simulate), 10) self.assertGreater(skipped_due_to_mag, 0) log_file_name = tempfile.mktemp(dir=self.output_dir, suffix='log.txt') alert_gen = AlertDataGenerator(testing=True) alert_gen.subdivide_obs(self.obs_list, htmid_level=6) for htmid in alert_gen.htmid_list: alert_gen.alert_data_from_htmid(htmid, star_db, photometry_class=TestAlertsVarCat, output_prefix='alert_test', output_dir=self.output_dir, dmag_cutoff=dmag_cutoff, log_file_name=log_file_name) dummy_sed = Sed() bp_dict = BandpassDict.loadTotalBandpassesFromFiles() phot_params = PhotometricParameters() # First, verify that the contents of the sqlite files are all correct n_tot_simulated = 0 alert_query = 'SELECT alert.uniqueId, alert.obshistId, meta.TAI, ' alert_query += 'meta.band, quiescent.flux, alert.dflux, ' alert_query += 'quiescent.snr, alert.snr, ' alert_query += 'alert.ra, alert.dec, alert.chipNum, ' alert_query += 'alert.xPix, alert.yPix, ast.pmRA, ast.pmDec, ' alert_query += 'ast.parallax ' alert_query += 'FROM alert_data AS alert ' alert_query += 'INNER JOIN metadata AS meta ON meta.obshistId=alert.obshistId ' alert_query += 'INNER JOIN quiescent_flux AS quiescent ' alert_query += 'ON quiescent.uniqueId=alert.uniqueId ' alert_query += 'AND quiescent.band=meta.band ' alert_query += 'INNER JOIN baseline_astrometry AS ast ' alert_query += 'ON ast.uniqueId=alert.uniqueId' alert_dtype = np.dtype([('uniqueId', int), ('obshistId', int), ('TAI', float), ('band', int), ('q_flux', float), ('dflux', float), ('q_snr', float), ('tot_snr', float), ('ra', float), ('dec', float), ('chipNum', int), ('xPix', float), ('yPix', float), ('pmRA', float), ('pmDec', float), ('parallax', float)]) sqlite_file_list = os.listdir(self.output_dir) n_tot_simulated = 0 obshistid_unqid_simulated_set = set() for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') alert_data = alert_db.execute_arbitrary(alert_query, dtype=alert_dtype) if len(alert_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=alert_data['TAI']) for i_obj in range(len(alert_data)): n_tot_simulated += 1 obshistid_unqid_simulated_set.add((alert_data['uniqueId'][i_obj] << obshistid_bits) + alert_data['obshistId'][i_obj]) unq = alert_data['uniqueId'][i_obj] obj_dex = (unq//1024)-1 self.assertAlmostEqual(self.pmra_truth[obj_dex], 0.001*alert_data['pmRA'][i_obj], 4) self.assertAlmostEqual(self.pmdec_truth[obj_dex], 0.001*alert_data['pmDec'][i_obj], 4) self.assertAlmostEqual(self.px_truth[obj_dex], 0.001*alert_data['parallax'][i_obj], 4) ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) distance_arcsec = 3600.0*distance msg = '\ntruth: %e %e\nalert: %e %e\n' % (ra_truth, dec_truth, alert_data['ra'][i_obj], alert_data['dec'][i_obj]) self.assertLess(distance_arcsec, 0.0005, msg=msg) obs = obs_dict[alert_data['obshistId'][i_obj]] chipname = chipNameFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, band=obs.bandpass) chipnum = int(chipname.replace('R', '').replace('S', ''). replace(' ', '').replace(';', '').replace(',', ''). replace(':', '')) self.assertEqual(chipnum, alert_data['chipNum'][i_obj]) xpix, ypix = pixelCoordsFromRaDecLSST(self.ra_truth[obj_dex], self.dec_truth[obj_dex], pm_ra=self.pmra_truth[obj_dex], pm_dec=self.pmdec_truth[obj_dex], parallax=self.px_truth[obj_dex], v_rad=self.vrad_truth[obj_dex], obs_metadata=obs, band=obs.bandpass) self.assertAlmostEqual(alert_data['xPix'][i_obj], xpix, 4) self.assertAlmostEqual(alert_data['yPix'][i_obj], ypix, 4) dmag_sim = -2.5*np.log10(1.0+alert_data['dflux'][i_obj]/alert_data['q_flux'][i_obj]) self.assertAlmostEqual(true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]], dmag_sim, 3) mag_name = ('u', 'g', 'r', 'i', 'z', 'y')[alert_data['band'][i_obj]] m5 = obs.m5[mag_name] q_mag = dummy_sed.magFromFlux(alert_data['q_flux'][i_obj]) self.assertAlmostEqual(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], q_mag, 4) snr, gamma = calcSNR_m5(self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex], bp_dict[mag_name], self.obs_mag_cutoff[alert_data['band'][i_obj]], phot_params) self.assertAlmostEqual(snr/alert_data['q_snr'][i_obj], 1.0, 4) tot_mag = self.mag0_truth_dict[alert_data['band'][i_obj]][obj_dex] + \ true_lc_dict[alert_data['uniqueId'][i_obj]][alert_data['obshistId'][i_obj]] snr, gamma = calcSNR_m5(tot_mag, bp_dict[mag_name], m5, phot_params) self.assertAlmostEqual(snr/alert_data['tot_snr'][i_obj], 1.0, 4) for val in obshistid_unqid_set: self.assertIn(val, obshistid_unqid_simulated_set) self.assertEqual(len(obshistid_unqid_set), len(obshistid_unqid_simulated_set)) astrometry_query = 'SELECT uniqueId, ra, dec, TAI ' astrometry_query += 'FROM baseline_astrometry' astrometry_dtype = np.dtype([('uniqueId', int), ('ra', float), ('dec', float), ('TAI', float)]) tai_list = [] for obs in self.obs_list: tai_list.append(obs.mjd.TAI) tai_list = np.array(tai_list) n_tot_ast_simulated = 0 for file_name in sqlite_file_list: if not file_name.endswith('db'): continue full_name = os.path.join(self.output_dir, file_name) self.assertTrue(os.path.exists(full_name)) alert_db = DBObject(full_name, driver='sqlite') astrometry_data = alert_db.execute_arbitrary(astrometry_query, dtype=astrometry_dtype) if len(astrometry_data) == 0: continue mjd_list = ModifiedJulianDate.get_list(TAI=astrometry_data['TAI']) for i_obj in range(len(astrometry_data)): n_tot_ast_simulated += 1 obj_dex = (astrometry_data['uniqueId'][i_obj]//1024) - 1 ra_truth, dec_truth = applyProperMotion(self.ra_truth[obj_dex], self.dec_truth[obj_dex], self.pmra_truth[obj_dex], self.pmdec_truth[obj_dex], self.px_truth[obj_dex], self.vrad_truth[obj_dex], mjd=mjd_list[i_obj]) distance = angularSeparation(ra_truth, dec_truth, astrometry_data['ra'][i_obj], astrometry_data['dec'][i_obj]) self.assertLess(3600.0*distance, 0.0005) del alert_gen gc.collect() self.assertGreater(n_tot_simulated, 10) self.assertGreater(len(obshistid_unqid_simulated_set), 10) self.assertLess(len(obshistid_unqid_simulated_set), n_total_observations) self.assertGreater(n_tot_ast_simulated, 0)
def test_flare_magnitudes_mixed_with_none(self): """ Test that we get the expected magnitudes out """ db = MLT_test_DB(database=self.db_name, driver='sqlite') # load the quiescent SEDs of the objects in our catalog sed_list = SedList(['lte028-5.0+0.5a+0.0.BT-Settl.spec.gz']*4, [17.1, 17.2, 17.3, 17.4], galacticAvList = [2.432, 1.876, 2.654, 2.364], fileDir=getPackageDir('sims_sed_library'), specMap=defaultSpecMap) bp_dict = BandpassDict.loadTotalBandpassesFromFiles() # calculate the quiescent fluxes of the objects in our catalog baseline_fluxes = bp_dict.fluxListForSedList(sed_list) bb_wavelen = np.arange(100.0, 1600.0, 0.1) bb = models.BlackBody(temperature=9000.0 * u.K, scale=1.0 * u.erg / (u.cm ** 2 * u.AA * u.s * u.sr)) bb_flambda = bb(bb_wavelen * u.nm).to_value() # this data is taken from the setUpClass() classmethod above t0_list = [456.2, 41006.2, 117.2, 10456.2] av_list = [2.432, 1.876, 2.654, 2.364] parallax_list = np.array([0.25, 0.15, 0.3, 0.22]) distance_list = 1.0/(206265.0*radiansFromArcsec(0.001*parallax_list)) distance_list *= 3.0857e18 # convert to cm dtype = np.dtype([('id', int), ('u', float), ('g', float)]) photParams = PhotometricParameters() ss = Sed() quiet_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_none_quiet_cat.txt') flare_cat_name = os.path.join(self.scratch_dir, 'mlt_mixed_with_none_flaring_cat.txt') # loop over several MJDs and verify that, to within a # milli-mag, our flaring model gives us the magnitudes # expected, given the light curves specified in # setUpClass() for mjd in (59580.0, 60000.0, 70000.0, 80000.0): obs = ObservationMetaData(mjd=mjd) quiet_cat = QuiescentCatalog(db, obs_metadata=obs) quiet_cat.write_catalog(quiet_cat_name) flare_cat = FlaringCatalog(db, obs_metadata=obs) flare_cat._mlt_lc_file = self.mlt_lc_name flare_cat.write_catalog(flare_cat_name) quiescent_data = np.genfromtxt(quiet_cat_name, dtype=dtype, delimiter=',') flaring_data = np.genfromtxt(flare_cat_name, dtype=dtype, delimiter=',') self.assertGreater(len(flaring_data), 3) for ix in range(len(flaring_data)): obj_id = flaring_data['id'][ix] self.assertEqual(obj_id, ix) # the models below are as specified in the # setUpClass() method if obj_id == 0 or obj_id == 1: amp = 1.0e42 dt = 3652.5 t_min = flare_cat._survey_start - t0_list[obj_id] tt = mjd - t_min while tt > dt: tt -= dt u_flux = amp*(1.0+np.power(np.sin(tt/100.0), 2)) g_flux = amp*(1.0+np.power(np.cos(tt/100.0), 2)) elif obj_id==2: amp = 2.0e41 dt = 365.25 t_min = flare_cat._survey_start - t0_list[obj_id] tt = mjd - t_min while tt > dt: tt -= dt u_flux = amp*(1.0+np.power(np.sin(tt/50.0), 2)) g_flux = amp*(1.0+np.power(np.cos(tt/50.0), 2)) else: u_flux = 0.0 g_flux = 0.0 # calculate the multiplicative effect of dust on a 9000K # black body bb_sed = Sed(wavelen=bb_wavelen, flambda=bb_flambda) u_bb_flux = bb_sed.calcFlux(bp_dict['u']) g_bb_flux = bb_sed.calcFlux(bp_dict['g']) a_x, b_x = bb_sed.setupCCM_ab() bb_sed.addDust(a_x, b_x, A_v=av_list[obj_id]) u_bb_dusty_flux = bb_sed.calcFlux(bp_dict['u']) g_bb_dusty_flux = bb_sed.calcFlux(bp_dict['g']) dust_u = u_bb_dusty_flux/u_bb_flux dust_g = g_bb_dusty_flux/g_bb_flux area = 4.0*np.pi*np.power(distance_list[obj_id], 2) tot_u_flux = baseline_fluxes[obj_id][0] + u_flux*dust_u/area tot_g_flux = baseline_fluxes[obj_id][1] + g_flux*dust_g/area msg = ('failed on object %d; mjd %.2f\n u_quiet %e u_flare %e\n g_quiet %e g_flare %e' % (obj_id, mjd, quiescent_data['u'][obj_id], flaring_data['u'][obj_id], quiescent_data['g'][obj_id], flaring_data['g'][obj_id])) self.assertEqual(quiescent_data['id'][obj_id], flaring_data['id'][obj_id], msg=msg) self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][0]), quiescent_data['u'][obj_id], 3, msg=msg) self.assertAlmostEqual(ss.magFromFlux(baseline_fluxes[obj_id][1]), quiescent_data['g'][obj_id], 3, msg=msg) self.assertAlmostEqual(ss.magFromFlux(tot_u_flux), flaring_data['u'][obj_id], 3, msg=msg) self.assertAlmostEqual(ss.magFromFlux(tot_g_flux), flaring_data['g'][obj_id], 3, msg=msg) if obj_id != 3: self.assertGreater(np.abs(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id]), 0.001, msg=msg) self.assertGreater(np.abs(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id]), 0.001, msg=msg) else: self.assertEqual(flaring_data['g'][obj_id]-quiescent_data['g'][obj_id], 0.0, msg=msg) self.assertEqual(flaring_data['u'][obj_id]-quiescent_data['u'][obj_id], 0.0, msg=msg) if os.path.exists(quiet_cat_name): os.unlink(quiet_cat_name) if os.path.exists(flare_cat_name): os.unlink(flare_cat_name)
def process_stellar_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, obs_md_list, proper_chip, variability_cache, out_data): t_start_chunk = time.time() #print('processing %d' % len(chunk)) ct_first = 0 ct_at_all = 0 ct_tot = 0 n_t = len(filter_obs) n_obj = len(chunk) coadd_visits = {} coadd_visits['u'] = 6 coadd_visits['g'] = 8 coadd_visits['r'] = 18 coadd_visits['i'] = 18 coadd_visits['z'] = 16 coadd_visits['y'] = 16 # from the overview paper # table 2; take m5 row and add Delta m5 row # to get down to airmass 1.2 m5_single = {} m5_single['u'] = 23.57 m5_single['g'] = 24.65 m5_single['r'] = 24.21 m5_single['i'] = 23.79 m5_single['z'] = 23.21 m5_single['y'] = 22.31 gamma_coadd = {} for bp in 'ugrizy': gamma_coadd[bp] = None gamma_single = {} for bp in 'ugrizy': gamma_single[bp] = [None] * n_t dflux = np.zeros((n_obj, n_t), dtype=float) dflux_for_mlt(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_kepler(chunk, filter_obs, mjd_obs, variability_cache, dflux) dflux_for_rrly(chunk, filter_obs, mjd_obs, variability_cache, dflux) dummy_sed = Sed() lsst_bp = BandpassDict.loadTotalBandpassesFromFiles() flux_q = np.zeros((6, n_obj), dtype=float) flux_coadd = np.zeros((6, n_obj), dtype=float) mag_coadd = np.zeros((6, n_obj), dtype=float) snr_coadd = np.zeros((6, n_obj), dtype=float) snr_single = {} snr_single_mag_grid = np.arange(14.0, 30.0, 0.05) phot_params_single = PhotometricParameters(nexp=1, exptime=30.0) t_start_snr = time.time() photometry_mask = np.zeros((n_obj, n_t), dtype=bool) photometry_mask_1d = np.zeros(n_obj, dtype=bool) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) phot_params_coadd = PhotometricParameters(nexp=1, exptime=30.0 * coadd_visits[bp]) flux_q[i_bp] = dummy_sed.fluxFromMag(chunk['%smag' % bp]) dflux_sub = dflux[:, valid_obs[0]] assert dflux_sub.shape == (n_obj, len(valid_obs[0])) dflux_mean = np.mean(dflux_sub, axis=1) assert dflux_mean.shape == (n_obj, ) flux_coadd[i_bp] = flux_q[i_bp] + dflux_mean mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp]) (snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp], coadd_m5[bp], phot_params_coadd) (snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid, lsst_bp[bp], m5_single[bp], phot_params_single) #print('got all snr in %e' % (time.time()-t_start_snr)) t_start_obj = time.time() snr_arr = np.zeros((n_obj, n_t), dtype=float) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs == i_bp) if len(valid_obs[0]) == 0: continue n_bp = len(valid_obs[0]) dflux_bp = dflux[:, valid_obs[0]] flux0_arr = flux_q[i_bp] flux_tot = flux0_arr[:, None] + dflux_bp mag_tot = dummy_sed.magFromFlux(flux_tot) snr_single_val = np.interp(mag_tot, snr_single_mag_grid, snr_single[bp]) noise_coadd = flux_coadd[i_bp] / snr_coadd[i_bp] noise_single = flux_tot / snr_single_val noise = np.sqrt(noise_coadd[:, None]**2 + noise_single**2) dflux_thresh = 5.0 * noise dflux_bp = np.abs(dflux_bp) detected = (dflux_bp >= dflux_thresh) snr_arr[:, valid_obs[0]] = dflux_bp / noise for i_obj in range(n_obj): if detected[i_obj].any(): photometry_mask_1d[i_obj] = True photometry_mask[i_obj, valid_obs[0]] = detected[i_obj] t_before_chip = time.time() chip_mask = apply_focal_plane(chunk['ra'], chunk['decl'], photometry_mask_1d, obs_md_list, filter_obs, proper_chip) duration = (time.time() - t_before_chip) / 3600.0 for i_obj in range(n_obj): if photometry_mask_1d[i_obj]: detected = photometry_mask[i_obj, :] & chip_mask[i_obj, :] if detected.any(): unq = chunk['simobjid'][i_obj] first_dex = np.where(detected)[0].min() out_data[unq] = (mjd_obs[first_dex], snr_arr[i_obj, first_dex], chunk['var_type'][i_obj]) if detected[0]: ct_first += 1 else: ct_at_all += 1
for lc_id in np.unique(lc_int_dex): valid_stars = np.where(lc_int_dex == lc_id) if len(valid_stars[0]) == 0: continue local_flux_factor = flux_factor[valid_stars] local_dmag_max = np.zeros(len(valid_stars[0]), dtype=float) local_mag_min = 41.0 * np.ones(len(valid_stars[0]), dtype=float) for mag_name in mag_name_list: mag_0 = chunk['%smag' % mag_name][valid_stars] local_dust_factor = dust_factor[mag_name][valid_stars] flux_0 = dummy_sed.fluxFromMag(mag_0) dflux = dflux_dict[lc_id][ mag_name] * local_dust_factor * local_flux_factor total_flux = flux_0 + dflux mag = dummy_sed.magFromFlux(total_flux) dmag = 2.5 * np.log10(1.0 + dflux / flux_0) local_dmag_max = np.where(dmag > local_dmag_max, dmag, local_dmag_max) local_mag_min = np.where(mag < local_mag_min, mag, local_mag_min) dmag_max[valid_stars] = local_dmag_max mag_min[valid_stars] = local_mag_min ct_total += len(chunk) with open(out_name, 'a') as out_file: for i_star in range(len(chunk)): if mag_min[i_star] > 24.89 or dmag_max[i_star] < 0.001: ct_bad += 1 out_file.write(
uflare, gflare, rflare, iflare, zflare, yflare, tpeak) = light_curve_from_class(flare_type, 2.0, rng) uflare *= flux_factor[i_star] gflare *= flux_factor[i_star] rflare *= flux_factor[i_star] iflare *= flux_factor[i_star] zflare *= flux_factor[i_star] yflare *= flux_factor[i_star] uflux = np.interp(tpeak, time, uflare) gflux = np.interp(tpeak, time, gflare) rflux = np.interp(tpeak, time, rflare) iflux = np.interp(tpeak, time, iflare) zflux = np.interp(tpeak, time, zflare) yflux = np.interp(tpeak, time, yflare) du += list(data['umag'][i_star] - ss.magFromFlux(base_u[i_star]+uflux)) dg += list(data['gmag'][i_star] - ss.magFromFlux(base_g[i_star]+gflux)) dr += list(data['rmag'][i_star] - ss.magFromFlux(base_r[i_star]+rflux)) di += list(data['imag'][i_star] - ss.magFromFlux(base_i[i_star]+iflux)) dz += list(data['zmag'][i_star] - ss.magFromFlux(base_z[i_star]+zflux)) dy += list(data['ymag'][i_star] - ss.magFromFlux(base_y[i_star]+yflux)) with open('delta_m_data.txt', 'w') as output_file: for uu, gg, rr, ii, zz, yy in zip(du, dg, dr, di, dz, dy): output_file.write('%e %e %e %e %e %e\n' % (uu, gg, rr, ii, zz, yy))
def _light_curves_from_query(self, cat_dict, query_result, grp, lc_per_field=None): t_dict = {} gamma_dict = {} m5_dict = {} t_min = None t_max = None for bp_name in cat_dict: self.lsstBandpassDict[bp_name].sbTophi() # generate a 2-D numpy array containing MJDs, m5, and photometric gamma values # for each observation in the given bandpass raw_array = np.array([[obs.mjd.TAI, obs.m5[bp_name], calcGamma(self.lsstBandpassDict[bp_name], obs.m5[obs.bandpass], self.phot_params)] for obs in grp if obs.bandpass == bp_name]).transpose() if len(raw_array) > 0: t_dict[bp_name] = raw_array[0] m5_dict[bp_name] = raw_array[1] gamma_dict[bp_name] = raw_array[2] local_t_min = t_dict[bp_name].min() local_t_max = t_dict[bp_name].max() if t_min is None or local_t_min < t_min: t_min = local_t_min if t_max is None or local_t_max > t_max: t_max = local_t_max snobj = SNObject() cat = cat_dict[list(cat_dict.keys())[0]] # does not need to be associated with a bandpass dummy_sed = Sed() n_actual_sn = 0 # how many SN have we actually delivered? for chunk in query_result: if lc_per_field is not None and n_actual_sn >= lc_per_field: break t_start_chunk = time.time() for sn in cat.iter_catalog(query_cache=[chunk]): sn_rng = self.sn_universe.getSN_rng(sn[1]) sn_t0 = self.sn_universe.drawFromT0Dist(sn_rng) if sn[5] <= self.z_cutoff and np.isfinite(sn_t0) and \ sn_t0 < t_max + cat.maxTimeSNVisible and \ sn_t0 > t_min - cat.maxTimeSNVisible: sn_c = self.sn_universe.drawFromcDist(sn_rng) sn_x1 = self.sn_universe.drawFromx1Dist(sn_rng) sn_x0 = self.sn_universe.drawFromX0Dist(sn_rng, sn_x1, sn_c, sn[4]) snobj.set(t0=sn_t0, c=sn_c, x1=sn_x1, x0=sn_x0, z=sn[5]) for bp_name in t_dict: t_list = t_dict[bp_name] m5_list = m5_dict[bp_name] gamma_list = gamma_dict[bp_name] bandpass = self.lsstBandpassDict[bp_name] if len(t_list) == 0: continue if snobj.maxtime() >= t_list[0] and snobj.mintime() <= t_list[-1]: active_dexes = np.where(np.logical_and(t_list >= snobj.mintime(), t_list <= snobj.maxtime())) t_active = t_list[active_dexes] m5_active = m5_list[active_dexes] gamma_active = gamma_list[active_dexes] if len(t_active) > 0: wave_ang = bandpass.wavelen*10.0 mask = np.logical_and(wave_ang > snobj.minwave(), wave_ang < snobj.maxwave()) wave_ang = wave_ang[mask] snobj.set(mwebv=sn[6]) sn_ff_buffer = snobj.flux(time=t_active, wave=wave_ang)*10.0 flambda_grid = np.zeros((len(t_active), len(bandpass.wavelen))) for ff, ff_sn in zip(flambda_grid, sn_ff_buffer): ff[mask] = np.where(ff_sn > 0.0, ff_sn, 0.0) fnu_grid = flambda_grid*bandpass.wavelen* \ bandpass.wavelen*dummy_sed._physParams.nm2m* \ dummy_sed._physParams.ergsetc2jansky/dummy_sed._physParams.lightspeed flux_list = \ (fnu_grid*bandpass.phi).sum(axis=1)*(bandpass.wavelen[1]-bandpass.wavelen[0]) acceptable = np.where(flux_list>0.0) flux_error_list = flux_list[acceptable]/ \ calcSNR_m5(dummy_sed.magFromFlux(flux_list[acceptable]), bandpass, m5_active[acceptable], self.phot_params, gamma=gamma_active[acceptable]) if len(acceptable) > 0: n_actual_sn += 1 if lc_per_field is not None and n_actual_sn > lc_per_field: break if sn[0] not in self.truth_dict: self.truth_dict[sn[0]] = {} self.truth_dict[sn[0]]['t0'] = sn_t0 self.truth_dict[sn[0]]['x1'] = sn_x1 self.truth_dict[sn[0]]['x0'] = sn_x0 self.truth_dict[sn[0]]['c'] = sn_c self.truth_dict[sn[0]]['z'] = sn[5] self.truth_dict[sn[0]]['E(B-V)'] = sn[6] if sn[0] not in self.mjd_dict: self.mjd_dict[sn[0]] = {} self.bright_dict[sn[0]] = {} self.sig_dict[sn[0]] = {} if bp_name not in self.mjd_dict[sn[0]]: self.mjd_dict[sn[0]][bp_name] = [] self.bright_dict[sn[0]][bp_name] = [] self.sig_dict[sn[0]][bp_name] = [] for tt, ff, ee in zip(t_active[acceptable], flux_list[acceptable], flux_error_list[0]): self.mjd_dict[sn[0]][bp_name].append(tt) self.bright_dict[sn[0]][bp_name].append(ff/3631.0) self.sig_dict[sn[0]][bp_name].append(ee/3631.0) print("chunk of ", len(chunk), " took ", time.time()-t_start_chunk)
def process_agn_chunk(chunk, filter_obs, mjd_obs, m5_obs, coadd_m5, m5_single, obs_md_list, proper_chip, out_data, lock): t_start_chunk = time.time() #print('processing %d' % len(chunk)) ct_first = 0 ct_at_all = 0 ct_tot = 0 n_t = len(filter_obs) n_obj = len(chunk) agn_model = ExtraGalacticVariabilityModels() dust_model = EBVbase() with h5py.File('data/ebv_grid.h5', 'r') as in_file: ebv_grid = in_file['ebv_grid'].value extinction_grid = in_file['extinction_grid'].value coadd_visits = {} coadd_visits['u'] = 6 coadd_visits['g'] = 8 coadd_visits['r'] = 18 coadd_visits['i'] = 18 coadd_visits['z'] = 16 coadd_visits['y'] = 16 gamma_coadd = {} for bp in 'ugrizy': gamma_coadd[bp] = None gamma_single = {} for bp in 'ugrizy': gamma_single[bp] = [None]*n_t params = {} params['agn_sfu'] = chunk['agn_sfu'] params['agn_sfg'] = chunk['agn_sfg'] params['agn_sfr'] = chunk['agn_sfr'] params['agn_sfi'] = chunk['agn_sfi'] params['agn_sfz'] = chunk['agn_sfz'] params['agn_sfy'] = chunk['agn_sfy'] params['agn_tau'] = chunk['agn_tau'] params['seed'] = chunk['id']+1 ebv = dust_model.calculateEbv(equatorialCoordinates=np.array([np.radians(chunk['ra']), np.radians(chunk['dec'])]), interp=True) for i_bp, bp in enumerate('ugrizy'): extinction_values = np.interp(ebv, ebv_grid, extinction_grid[i_bp]) chunk['%s_ab' % bp] += extinction_values chunk['AGNLSST%s' % bp] += extinction_values dmag = agn_model.applyAgn(np.where(np.array([True]*len(chunk))), params, mjd_obs, redshift=chunk['redshift']) dmag_mean = np.mean(dmag, axis=2) assert dmag_mean.shape == (6,n_obj) dummy_sed = Sed() lsst_bp = BandpassDict.loadTotalBandpassesFromFiles() flux_gal = np.zeros((6,n_obj), dtype=float) flux_agn_q = np.zeros((6,n_obj), dtype=float) flux_coadd = np.zeros((6,n_obj), dtype=float) mag_coadd = np.zeros((6,n_obj), dtype=float) snr_coadd = np.zeros((6,n_obj), dtype=float) snr_single = {} snr_single_mag_grid = np.arange(14.0, 30.0, 0.05) phot_params_single = PhotometricParameters(nexp=1, exptime=30.0) t_start_snr = time.time() for i_bp, bp in enumerate('ugrizy'): phot_params_coadd = PhotometricParameters(nexp=1, exptime=30.0*coadd_visits[bp]) flux_gal[i_bp] = dummy_sed.fluxFromMag(chunk['%s_ab' % bp]) flux_agn_q[i_bp] = dummy_sed.fluxFromMag(chunk['AGNLSST%s' % bp] + dmag_mean[i_bp,:]) flux_coadd[i_bp] = flux_gal[i_bp]+flux_agn_q[i_bp] mag_coadd[i_bp] = dummy_sed.magFromFlux(flux_coadd[i_bp]) (snr_coadd[i_bp], gamma) = SNR.calcSNR_m5(mag_coadd[i_bp], lsst_bp[bp], coadd_m5[bp], phot_params_coadd) (snr_single[bp], gamma) = SNR.calcSNR_m5(snr_single_mag_grid, lsst_bp[bp], m5_single[bp], phot_params_single) #print('got all snr in %e' % (time.time()-t_start_snr)) t_start_obj = time.time() photometry_mask = np.zeros((n_obj, n_t), dtype=bool) photometry_mask_1d = np.zeros(n_obj, dtype=bool) snr_arr = np.zeros((n_obj, n_t), dtype=float) for i_bp, bp in enumerate('ugrizy'): valid_obs = np.where(filter_obs==i_bp) n_bp = len(valid_obs[0]) if n_bp == 0: continue mag0_arr = chunk['AGNLSST%s' % bp] dmag_bp = dmag[i_bp][:,valid_obs[0]] assert dmag_bp.shape == (n_obj, n_bp) agn_flux_tot = dummy_sed.fluxFromMag(mag0_arr[:,None]+dmag_bp) q_flux = flux_agn_q[i_bp] agn_dflux = np.abs(agn_flux_tot-q_flux[:,None]) flux_tot = flux_gal[i_bp][:, None] + agn_flux_tot assert flux_tot.shape == (n_obj, n_bp) mag_tot = dummy_sed.magFromFlux(flux_tot) snr_single_val = np.interp(mag_tot, snr_single_mag_grid, snr_single[bp]) noise_coadd = flux_coadd[i_bp]/snr_coadd[i_bp] noise_single = flux_tot/snr_single_val noise = np.sqrt(noise_coadd[:,None]**2 + noise_single**2) dflux_thresh = 5.0*noise detected = (agn_dflux>=dflux_thresh) assert detected.shape == (n_obj, n_bp) snr_arr[:,valid_obs[0]] = agn_dflux/noise for i_obj in range(n_obj): if detected[i_obj].any(): photometry_mask_1d[i_obj] = True photometry_mask[i_obj, valid_obs[0]] = detected[i_obj] t_before_chip = time.time() chip_mask = apply_focal_plane(chunk['ra'], chunk['dec'], photometry_mask_1d, obs_md_list, filter_obs, proper_chip) duration = (time.time()-t_before_chip)/3600.0 unq_out = -1*np.ones(n_obj, dtype=int) mjd_out = -1.0*np.ones(n_obj, dtype=float) snr_out = -1.0*np.ones(n_obj, dtype=float) for i_obj in range(n_obj): if photometry_mask_1d[i_obj]: detected = photometry_mask[i_obj,:] & chip_mask[i_obj,:] if detected.any(): unq_out[i_obj] = chunk['galtileid'][i_obj] first_dex = np.where(detected)[0].min() mjd_out[i_obj] = mjd_obs[first_dex] snr_out[i_obj] = snr_arr[i_obj, first_dex] valid = np.where(unq_out>=0) unq_out = unq_out[valid] mjd_out = mjd_out[valid] snr_out = snr_out[valid] with lock: existing_keys = list(out_data.keys()) if len(existing_keys) == 0: key_val = 0 else: key_val = max(existing_keys) while key_val in existing_keys: key_val += 1 out_data[key_val] = (None, None, None) out_data[key_val] = (unq_out, mjd_out, snr_out)