def setUpClass(cls): #NB This is basically nonsense, we're just selecting any old #extracted source so we can check the feature extraction syntax. xtrsrc_qry = """SELECT id as trigger_xtrsrc FROM extractedsource""" cursor = execute(xtrsrc_qry) cls.transients = get_db_rows_as_dicts(cursor)
def get_assoc_entries(db, runcat_id): """ Return the full history of variability indices for a runcat entry, ordered by time. """ query = """\ select a.runcat ,a.xtrsrc ,x.extract_type ,i.taustart_ts ,a.v_int ,a.eta_int ,a.f_datapoints ,a.type ,r.mon_src from assocxtrsource a ,extractedsource x ,image i ,runningcatalog r where a.xtrsrc = x.id and x.image = i.id and a.runcat = r.id and r.id = %(runcat_id)s order by i.taustart_ts """ db.cursor.execute(query, {'runcat_id': runcat_id}) return get_db_rows_as_dicts(db.cursor)
def get_sources_filtered_by_final_variability(dataset_id, eta_min, v_min, # minpoints ): """ Search the database to find high-variability lightcurves. Uses the variability associated with the last datapoint in a lightcurve as the key criteria. Args: dataset_id (int): Dataset to search eta_min (float): Minimum value of eta-index to return. v_min (float): Minimum value of V-index to return. Returns: list: (list of dicts) Each dict represents a runningcatalog_flux entry matching the filter criteria. """ query = """\ SELECT rc.id as runcat_id ,image.band ,ax.v_int ,ax.eta_int FROM runningcatalog as rc JOIN assocxtrsource as ax ON ax.runcat = rc.id JOIN extractedsource as ex ON ax.xtrsrc = ex.id JOIN image ON ex.image = image.id JOIN ( -- Determine which are the most recent variability values -- for each lightcurve. SELECT a.runcat as runcat_id, i.band as band, max(i.taustart_ts) as MaxTimestamp FROM assocxtrsource a JOIN extractedsource e ON a.xtrsrc = e.id JOIN image i ON e.image = i.id GROUP BY runcat_id, band ) last_timestamps ON rc.id = last_timestamps.runcat_id AND image.band = last_timestamps.band AND image.taustart_ts = last_timestamps.MaxTimestamp WHERE rc.dataset = %(dataset_id)s AND eta_int >= %(eta_min)s AND v_int >= %(v_min)s """ cursor = tkp.db.Database().cursor cursor.execute(query, {'dataset_id': dataset_id, 'eta_min':eta_min, 'v_min':v_min, }) transients = get_db_rows_as_dicts(cursor) return transients
def get_sources_filtered_by_final_variability(dataset_id, eta_min, v_min, # minpoints ): """ Search the database to find high-variability lightcurves. Uses the variability associated with the last datapoint in a lightcurve as the key criteria. Args: dataset_id (int): Dataset to search eta_min (float): Minimum value of eta-index to return. v_min (float): Minimum value of V-index to return. Returns: list: (list of dicts) Each dict represents a runningcatalog_flux entry matching the filter criteria. """ query = """\ SELECT rc.id as runcat_id ,image.band ,ax.v_int ,ax.eta_int FROM runningcatalog as rc JOIN assocxtrsource as ax ON ax.runcat = rc.id JOIN extractedsource as ex ON ax.xtrsrc = ex.id JOIN image ON ex.image = image.id JOIN ( -- Determine which are the most recent variability values -- for each lightcurve. SELECT a.runcat as runcat_id, i.band as band, max(i.taustart_ts) as MaxTimestamp FROM assocxtrsource a JOIN extractedsource e ON a.xtrsrc = e.id JOIN image i ON e.image = i.id GROUP BY runcat_id, band ) last_timestamps ON rc.id = last_timestamps.runcat_id AND image.band = last_timestamps.band AND image.taustart_ts = last_timestamps.MaxTimestamp WHERE rc.dataset = %(dataset_id)s AND eta_int >= %(eta_min)s AND v_int >= %(v_min)s """ cursor = tkp.db.Database().connection.cursor() cursor.execute(query, {'dataset_id': dataset_id, 'eta_min':eta_min, 'v_min':v_min, }) transients = get_db_rows_as_dicts(cursor) return transients
def test_full_transient_search_routine(self): bands = self.dataset.frequency_bands() self.assertEqual(len(bands), 1) # First run with lax limits: transients = multi_epoch_transient_search( eta_lim=1.1, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1 ) self.assertEqual(len(transients), 3) qry = """\ SELECT tr.* FROM transient tr ,runningcatalog rc WHERE rc.dataset = %(dsid)s AND tr.runcat = rc.id """ cursor = self.database.connection.cursor() cursor.execute(qry, {"dsid": self.dataset.id}) transient_table_entries = get_db_rows_as_dicts(cursor) self.assertEqual(len(transient_table_entries), len(transients)) # for t in all_transients: # print "V_int:", t['v_int'], " eta_int:", t['eta_int'] # Now test thresholding: more_highly_variable = sum(t["v_int"] > 2.0 for t in transients) very_non_flat = sum(t["eta_int"] > 100.0 for t in transients) transients = multi_epoch_transient_search( eta_lim=1.1, V_lim=2.0, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1 ) self.assertEqual(len(transients), more_highly_variable) transients = multi_epoch_transient_search( eta_lim=100, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1 ) self.assertEqual(len(transients), very_non_flat)
def test_two_field_overlap_new_transient(self): """Now for something more interesting - two overlapping fields, 4 sources: one steady source only in lower field, one steady source in both fields, one steady source only in upper field, one transient source in both fields but only at 2nd timestep. """ n_images = 2 xtr_radius = 1.5 im_params = db_subs.example_dbimage_datasets(n_images, xtr_radius=xtr_radius) im_params[1]['centre_decl'] += xtr_radius * 1 imgs = [] lower_steady_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius) upper_steady_src = db_subs.example_extractedsource_tuple( ra=im_params[1]['centre_ra'], dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius) overlap_steady_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius) overlap_transient = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius) imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0])) imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1])) imgs[0].insert_extracted_sources([lower_steady_src, overlap_steady_src]) nd_posns = dbmon.get_nulldetections(imgs[0].id, deRuiter_r=1) self.assertEqual(len(nd_posns), 0) imgs[0].associate_extracted_sources(deRuiter_r=0.1) imgs[1].insert_extracted_sources([upper_steady_src, overlap_steady_src, overlap_transient]) nd_posns = dbmon.get_nulldetections(imgs[1].id, deRuiter_r=1) self.assertEqual(len(nd_posns), 0) imgs[1].associate_extracted_sources(deRuiter_r=0.1) runcats = columns_from_table('runningcatalog', where={'dataset': self.dataset.id}) self.assertEqual(len(runcats), 4) #sanity check. monlist = columns_from_table('monitoringlist', where={'dataset': self.dataset.id}) self.assertEqual(len(monlist), 1) transients_qry = """\ SELECT * FROM transient tr ,runningcatalog rc WHERE rc.dataset = %s AND tr.runcat = rc.id """ self.database.cursor.execute(transients_qry, (self.dataset.id,)) transients = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(transients), 1)
def test_two_field_overlap_new_transient(self): """Now for something more interesting - two overlapping fields, 4 sources: one steady source only in lower field, one steady source in both fields, one steady source only in upper field, one transient source in both fields but only at 2nd timestep. """ n_images = 2 xtr_radius = 1.5 im_params = db_subs.generate_timespaced_dbimages_data( n_images, xtr_radius=xtr_radius) im_params[1]['centre_decl'] += xtr_radius * 1 imgs = [] lower_steady_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius) upper_steady_src = db_subs.example_extractedsource_tuple( ra=im_params[1]['centre_ra'], dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius) overlap_steady_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius) overlap_transient = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius) imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0])) imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1])) imgs[0].insert_extracted_sources( [lower_steady_src, overlap_steady_src]) imgs[0].associate_extracted_sources( deRuiter_r=0.1, new_source_sigma_margin=new_source_sigma_margin) nd_posns = dbnd.get_nulldetections(imgs[0].id) self.assertEqual(len(nd_posns), 0) imgs[1].insert_extracted_sources( [upper_steady_src, overlap_steady_src, overlap_transient]) imgs[1].associate_extracted_sources( deRuiter_r=0.1, new_source_sigma_margin=new_source_sigma_margin) nd_posns = dbnd.get_nulldetections(imgs[1].id) self.assertEqual(len(nd_posns), 0) runcats = columns_from_table('runningcatalog', where={'dataset': self.dataset.id}) self.assertEqual(len(runcats), 4) #sanity check. newsources_qry = """\ SELECT * FROM newsource tr ,runningcatalog rc WHERE rc.dataset = %s AND tr.runcat = rc.id """ self.database.cursor.execute(newsources_qry, (self.dataset.id, )) newsources = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(newsources), 1)
def get_newsources_for_dataset(dsid): """ Returns dicts representing all newsources for this dataset. Args: dsid: Dataset id Returns: list: (list of dicts) Each dict represents one newsource. The dict keys are all the columns in the newsources table, plus the 'taustart_ts' from the image table, which represents the trigger time. """ qry = """\ SELECT tr.id ,tr.previous_limits_image ,rc.id as runcat_id ,img.taustart_ts ,img.band ,ax.v_int ,ax.eta_int , ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_min) / limits_image.rms_min) AS low_thresh_sigma , ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_max) / limits_image.rms_max) AS high_thresh_sigma FROM newsource tr ,runningcatalog rc ,extractedsource ex ,image img ,assocxtrsource ax ,image limits_image WHERE rc.dataset = %(dsid)s AND tr.runcat = rc.id AND tr.trigger_xtrsrc = ex.id AND ex.image = img.id AND ax.runcat = rc.id AND ax.xtrsrc = ex.id AND tr.previous_limits_image = limits_image.id """ cursor = Database().cursor cursor.execute(qry, {'dsid':dsid}) newsource_rows_for_dataset = get_db_rows_as_dicts(cursor) return newsource_rows_for_dataset
def get_newsources_for_dataset(dsid): """ Returns dicts representing all newsources for this dataset. Args: dsid: Dataset id Returns: list: (list of dicts) Each dict represents one newsource. The dict keys are all the columns in the newsources table, plus the 'taustart_ts' from the image table, which represents the trigger time. """ qry = """\ SELECT tr.id ,tr.previous_limits_image ,rc.id as runcat_id ,img.taustart_ts ,img.band ,ax.v_int ,ax.eta_int , ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_min) / limits_image.rms_min) AS low_thresh_sigma , ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_max) / limits_image.rms_max) AS high_thresh_sigma FROM newsource tr ,runningcatalog rc ,extractedsource ex ,image img ,assocxtrsource ax ,image limits_image WHERE rc.dataset = %(dsid)s AND tr.runcat = rc.id AND tr.trigger_xtrsrc = ex.id AND ex.image = img.id AND ax.runcat = rc.id AND ax.xtrsrc = ex.id AND tr.previous_limits_image = limits_image.id """ cursor = Database().connection.cursor() cursor.execute(qry, {'dsid':dsid}) newsource_rows_for_dataset = get_db_rows_as_dicts(cursor) return newsource_rows_for_dataset
def test_two_field_basic_case(self): """ Here we create 2 disjoint image fields, with one source at centre of each, and check that the second source inserted does not get flagged as newsource. """ n_images = 2 xtr_radius = 1.5 im_params = db_subs.generate_timespaced_dbimages_data( n_images, xtr_radius=xtr_radius) im_params[1]['centre_decl'] += xtr_radius * 2 + 0.5 imgs = [] for idx in range(len(im_params)): imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx])) for idx in range(len(im_params)): central_src = db_subs.example_extractedsource_tuple( ra=im_params[idx]['centre_ra'], dec=im_params[idx]['centre_decl']) imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx])) imgs[idx].insert_extracted_sources([central_src]) imgs[idx].associate_extracted_sources(deRuiter_r, new_source_sigma_margin) runcats = columns_from_table('runningcatalog', where={'dataset': self.dataset.id}) self.assertEqual(len(runcats), 2) #Just a sanity check. newsources_qry = """\ SELECT * FROM newsource tr ,runningcatalog rc WHERE rc.dataset = %s AND tr.runcat = rc.id """ self.database.cursor.execute(newsources_qry, (self.dataset.id, )) newsources = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(newsources), 0)
def per_timestep_variability_indices(db, runcat_id): query = """\ select a.runcat ,a.xtrsrc ,i.taustart_ts ,a.v_int ,a.eta_int from assocxtrsource a ,extractedsource x ,image i ,runningcatalog r where a.xtrsrc = x.id and x.image = i.id and a.runcat = r.id and r.id = %(runcat_id)s order by r.wm_ra ,i.taustart_ts """ db.cursor.execute(query, {'runcat_id': runcat_id}) return get_db_rows_as_dicts(db.cursor)
def test_two_field_basic_case(self): """ Here we create 2 disjoint image fields, with one source at centre of each, and check that the second source inserted does not get flagged as newsource. """ n_images = 2 xtr_radius = 1.5 im_params = db_subs.generate_timespaced_dbimages_data(n_images, xtr_radius=xtr_radius) im_params[1]['centre_decl'] += xtr_radius * 2 + 0.5 imgs = [] for idx in range(len(im_params)): imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx])) for idx in range(len(im_params)): central_src = db_subs.example_extractedsource_tuple( ra=im_params[idx]['centre_ra'], dec=im_params[idx]['centre_decl']) imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx])) imgs[idx].insert_extracted_sources([central_src]) imgs[idx].associate_extracted_sources(deRuiter_r, new_source_sigma_margin) runcats = columns_from_table('runningcatalog', where={'dataset':self.dataset.id}) self.assertEqual(len(runcats), 2) #Just a sanity check. newsources_qry = """\ SELECT * FROM newsource tr ,runningcatalog rc WHERE rc.dataset = %s AND tr.runcat = rc.id """ self.database.cursor.execute(newsources_qry, (self.dataset.id,)) newsources = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(newsources), 0)
def test_lightcurve(self): # make 4 images with different date images = [] image_datasets = db_subs.generate_timespaced_dbimages_data(n_images=4, taustart_ts= datetime.datetime(2010, 3, 3) ) for dset in image_datasets: image = Image(dataset=self.dataset, data=dset) images.append(image) # 3 sources per image, with different coordinates & flux data_list = [] for i in range(1, 4): data_list.append({ 'ra': 111.11 + i, 'decl': 11.11 + i, 'i_peak': 10. * i , 'i_peak_err': 0.1, }) # Insert the 3 sources in each image, while further varying the flux lightcurves_sorted_by_ra = [[],[],[]] for im_idx, image in enumerate(images): # Create the "source finding results" # Note that we reuse 'i_peak' as both peak & integrated flux. img_sources = [] for src_idx, data in enumerate(data_list): src = db_subs.example_extractedsource_tuple( ra = data['ra'],dec=data['decl'], peak=data['i_peak']* (1 + im_idx), flux = data['i_peak']* (1 + im_idx) ) lightcurves_sorted_by_ra[src_idx].append(src) img_sources.append(src) insert_extracted_sources(image._id, img_sources) associate_extracted_sources(image._id, deRuiter_r=3.7, new_source_sigma_margin=3) # updates the dataset and its set of images self.dataset.update() self.dataset.update_images() # update the images and their sets of sources for image in self.dataset.images: image.update() image.update_sources() # Now pick last image, select the first source (smallest RA) # and extract its light curve sources = self.dataset.images[-1].sources sources = sorted(sources, key=attrgetter('ra')) lightcurve = ligtcurve_func(sources[0]._id) # check if the sources are associated in all images self.assertEqual(len(images), len(lightcurve)) self.assertEqual(lightcurve[0][0], datetime.datetime(2010, 3, 3, 0, 0)) self.assertEqual(lightcurve[1][0], datetime.datetime(2010, 3, 4, 0, 0)) self.assertEqual(lightcurve[2][0], datetime.datetime(2010, 3, 5, 0, 0)) self.assertEqual(lightcurve[3][0], datetime.datetime(2010, 3, 6, 0, 0)) self.assertAlmostEqual(lightcurve[0][2], 10.) self.assertAlmostEqual(lightcurve[1][2], 20.) self.assertAlmostEqual(lightcurve[2][2], 30.) self.assertAlmostEqual(lightcurve[3][2], 40.) #Check the summary statistics (avg flux, etc) query = """\ SELECT rf.avg_f_int ,rf.avg_f_int_sq ,avg_weighted_f_int ,avg_f_int_weight FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset)s AND r.id = rf.runcat ORDER BY r.wm_ra """ self.database.cursor.execute(query, {'dataset': self.dataset.id}) runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra)) for idx, flux_summary in enumerate(runcat_flux_entries): py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx]) for key in flux_summary.keys(): self.assertAlmostEqual(flux_summary[key], py_results[-1][key]) #Now check the per-timestep statistics (variability indices) sorted_runcat_ids = columns_from_table('runningcatalog', where={'dataset':self.dataset.id}, order='wm_ra') sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids] for idx, rcid in enumerate(sorted_runcat_ids): db_indices = db_queries.get_assoc_entries(self.database, rcid) py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx]) self.assertEqual(len(db_indices), len(py_indices)) for nstep in range(len(db_indices)): for key in ('v_int', 'eta_int', 'f_datapoints'): self.assertAlmostEqual(db_indices[nstep][key], py_indices[nstep][key], places=5)
def test_many2manyflux_reduced_to_two_1to1(self): """ (See also assoc. test test_many2many_reduced_to_two_1to1 ) In this test-case we cross-associate between a rhombus of sources spread about a central position, east-west in the first image, north-south in the second. The latter, north-south pair are slightly offset towards positive RA and negative RA respectively. The result is that the candidate associations are pruned down to two one-to-one pairings.. """ dataset = tkp.db.DataSet(database=self.database, data={ 'description': 'flux test set: n-m, ' + self._testMethodName }) n_images = 2 im_params = db_subs.generate_timespaced_dbimages_data(n_images) centre_ra, centre_dec = 123., 10.5, offset_deg = 20 / 3600. #20 arcsec tiny_offset_deg = 1 / 3600. #1 arcsec eastern_src = db_subs.example_extractedsource_tuple( ra=centre_ra + offset_deg, dec=centre_dec, peak=1.5, peak_err=1e-1, flux=3.0, flux_err=1e-1, ) western_src = db_subs.example_extractedsource_tuple( ra=centre_ra - offset_deg, dec=centre_dec, peak=1.7, peak_err=1e-1, flux=3.2, flux_err=1e-1, ) northern_source = db_subs.example_extractedsource_tuple( ra=centre_ra + tiny_offset_deg, dec=centre_dec + offset_deg, peak=1.8, peak_err=1e-1, flux=3.3, flux_err=1e-1, ) southern_source = db_subs.example_extractedsource_tuple( ra=centre_ra - tiny_offset_deg, dec=centre_dec - offset_deg, peak=1.4, peak_err=1e-1, flux=2.9, flux_err=1e-1, ) # image 1 image1 = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0]) dbgen.insert_extracted_sources(image1.id, [eastern_src, western_src], 'blind') associate_extracted_sources(image1.id, deRuiter_r=3.717) # image 2 image2 = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1]) dbgen.insert_extracted_sources(image2.id, [northern_source, southern_source], 'blind') associate_extracted_sources(image2.id, deRuiter_r=3.717) # Manually compose the lists of sources we expect to see associated # into runningcatalog entries: # NB img1_srclist[1] has larger RA value. lightcurves_sorted_by_ra = [] lightcurves_sorted_by_ra.append([western_src, southern_source]) lightcurves_sorted_by_ra.append([eastern_src, northern_source]) #Check the summary statistics (avg flux, etc) query = """\ SELECT rf.avg_f_int ,rf.avg_f_int_sq ,avg_weighted_f_int ,avg_f_int_weight FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset)s AND r.id = rf.runcat ORDER BY r.wm_ra, r.wm_decl """ self.database.cursor.execute(query, {'dataset': dataset.id}) runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra)) for idx, flux_summary in enumerate(runcat_flux_entries): py_results = db_subs.lightcurve_metrics( lightcurves_sorted_by_ra[idx]) for key in flux_summary.keys(): self.assertAlmostEqual(flux_summary[key], py_results[-1][key]) #Now check the per-timestep statistics (variability indices) sorted_runcat_ids = columns_from_table('runningcatalog', where={'dataset': dataset.id}, order='wm_ra,wm_decl') sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids] for idx, rcid in enumerate(sorted_runcat_ids): db_indices = db_queries.get_assoc_entries(self.database, rcid) py_indices = db_subs.lightcurve_metrics( lightcurves_sorted_by_ra[idx]) self.assertEqual(len(db_indices), len(py_indices)) for nstep in range(len(db_indices)): for key in ('v_int', 'eta_int', 'f_datapoints'): self.assertAlmostEqual(db_indices[nstep][key], py_indices[nstep][key])
def test_one2manyflux(self): dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: 1-n'}) n_images = 2 im_params = db_subs.generate_timespaced_dbimages_data(n_images) central_ra, central_dec = 123.1235, 10.55, position_offset_deg = 100. / 3600 #100 arcsec = 0.03 deg approx # image 1 image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0]) imageid1 = image.id img1_srclist = [] # 1 source img1_srclist.append( db_subs.example_extractedsource_tuple( central_ra, central_dec, peak=1.5, peak_err=5e-1, flux=3.0, flux_err=5e-1, )) dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind') associate_extracted_sources(imageid1, deRuiter_r=3.717) # image 2 image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1]) imageid2 = image.id img2_srclist = [] # 2 sources (both close to source 1, catching the 1-to-many case) img2_srclist.append( db_subs.example_extractedsource_tuple( central_ra, central_dec, peak=1.6, peak_err=5e-1, flux=3.2, flux_err=5e-1, )) img2_srclist.append( db_subs.example_extractedsource_tuple( central_ra + position_offset_deg, central_dec, peak=1.9, peak_err=5e-1, flux=3.4, flux_err=5e-1, )) dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind') associate_extracted_sources(imageid2, deRuiter_r=3.717) # Manually compose the lists of sources we expect to see associated # into runningcatalog entries: # NB img2_srclist[1] has larger RA value. lightcurves_sorted_by_ra = [] lightcurves_sorted_by_ra.append([img1_srclist[0], img2_srclist[0]]) lightcurves_sorted_by_ra.append([img1_srclist[0], img2_srclist[1]]) #Check the summary statistics (avg flux, etc) query = """\ SELECT rf.avg_f_int ,rf.avg_f_int_sq ,avg_weighted_f_int ,avg_f_int_weight FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset)s AND r.id = rf.runcat ORDER BY r.wm_ra """ self.database.cursor.execute(query, {'dataset': dataset.id}) runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(runcat_flux_entries), 2) for idx, flux_summary in enumerate(runcat_flux_entries): py_results = db_subs.lightcurve_metrics( lightcurves_sorted_by_ra[idx]) for key in flux_summary.keys(): self.assertAlmostEqual(flux_summary[key], py_results[-1][key]) #Now check the per-timestep statistics (variability indices) sorted_runcat_ids = columns_from_table('runningcatalog', where={'dataset': dataset.id}, order='wm_ra') sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids] for idx, rcid in enumerate(sorted_runcat_ids): db_indices = db_queries.get_assoc_entries(self.database, rcid) py_indices = db_subs.lightcurve_metrics( lightcurves_sorted_by_ra[idx]) self.assertEqual(len(db_indices), len(py_indices)) for nstep in range(len(db_indices)): for key in ('v_int', 'eta_int', 'f_datapoints'): self.assertAlmostEqual(db_indices[nstep][key], py_indices[nstep][key])
def _select_updated_variability_indices(image_id): """ Select sources and integrated flux variability indices, for sources which have had an extra datapoint added by the specified image. As part of the results we return a field 'new_transient' which is TRUE if the runcat-source/band combination does not yet have an entry in the transient table. NB Variability index = std.dev(flux) / mean(flux). In the pathological case where mean(flux)==0.0, we simply substitute variability index = std.dev(flux) / 1e-6 (1 microJansky) to avoid division by zero errors. (TO DO: Return indices based on peak fluxes as well.) *Args*: - image_id: the image and thus frequency band being searched for variability *Returns*: A list of dicts with keys as follows: [{ runcat, band, f_datapoints, wm_ra, wm_decl, wm_ra_err, wm_decl_err, v_int, eta_int, trigger_xtrsrc, new_transient }] """ # Note: We cannot trivially calculate an updated 'siglevel' probability, # and selecting it from transients gives the *old* value. # So; we recalculate it later, (using scipy.stats), # and apply a threshold there. # NB We also perform a left outer join with the transient table, # to determine if the source has been inserted into that table yet. # This allows us to distinguish newly identified transients. query = """\ SELECT t1.runcat ,t1.band ,t1.f_datapoints ,t1.wm_ra ,t1.wm_decl ,t1.wm_ra_err ,t1.wm_decl_err ,t1.V_int_inter / t1.avg_f_int AS v_int ,t1.eta_int_inter / t1.avg_f_int_weight AS eta_int ,CASE WHEN tr0.trigger_xtrsrc IS NULL THEN t1.xtrsrc ELSE tr0.trigger_xtrsrc END AS trigger_xtrsrc ,CASE WHEN tr0.trigger_xtrsrc IS NULL THEN TRUE ELSE FALSE END AS new_transient FROM (SELECT rf0.runcat ,rf0.band ,f_datapoints ,wm_ra ,wm_decl ,wm_ra_err ,wm_decl_err ,CASE WHEN avg_f_int = 0.0 THEN 0.000001 ELSE avg_f_int END AS avg_f_int ,avg_f_int_weight ,CASE WHEN rf0.f_datapoints = 1 THEN 0 WHEN avg_f_int_sq - avg_f_int * avg_f_int < 0 THEN 0 ELSE SQRT(CAST(rf0.f_datapoints AS DOUBLE PRECISION) * (avg_f_int_sq - avg_f_int * avg_f_int) / (CAST(rf0.f_datapoints AS DOUBLE PRECISION) - 1.0) ) END AS V_int_inter ,CASE WHEN rf0.f_datapoints = 1 THEN 0 ELSE (CAST(rf0.f_datapoints AS DOUBLE PRECISION) / (CAST(rf0.f_datapoints AS DOUBLE PRECISION) - 1.0)) * (avg_f_int_weight * avg_weighted_f_int_sq - avg_weighted_f_int * avg_weighted_f_int) END AS eta_int_inter ,a0.xtrsrc FROM runningcatalog rc0 ,runningcatalog_flux rf0 ,image i0 ,assocxtrsource a0 ,extractedsource x0 WHERE i0.id = %(imgid)s AND rc0.dataset = i0.dataset AND rc0.id = rf0.runcat AND rf0.band = i0.band AND rc0.id = a0.runcat AND a0.xtrsrc = x0.id AND x0.image = %(imgid)s ) t1 LEFT OUTER JOIN transient tr0 ON t1.runcat = tr0.runcat AND t1.band = tr0.band ORDER BY t1.runcat ,t1.band """ qry_params = {'imgid':image_id} cursor = tkp.db.execute(query, qry_params) results = get_db_rows_as_dicts(cursor) return results
def test_basic_case(self): im_params = self.im_params blind_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'], ) superimposed_mon_src = blind_src mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001) # Simulate a source that does not get fit, for good measure: mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.) #Sorted by increasing RA: mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field] mon_posns = [(m.ra, m.dec) for m in mon_srcs] dbgen.insert_monitor_positions(self.dataset.id,mon_posns) images = [] for img_pars in self.im_params: img = tkp.db.Image(dataset=self.dataset, data=img_pars) dbgen.insert_extracted_sources(img.id, [blind_src], 'blind') associate_extracted_sources(img.id, deRuiter_r=5.68) nd_requests = get_nulldetections(img.id) self.assertEqual(len(nd_requests),0) mon_requests = dbmon.get_monitor_entries(self.dataset.id) self.assertEqual(len(mon_requests),len(mon_srcs)) # mon requests is a list of tuples [(id,ra,decl)] # Ensure sorted by RA for cross-checking: mon_requests = sorted(mon_requests, key = lambda s: s[1]) for idx in range(len(mon_srcs)): self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra) self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec) #Insert fits for the in-field sources and then associate dbgen.insert_extracted_sources(img.id, [superimposed_mon_src, mon_src_in_field], 'ff_ms', ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]]) dbmon.associate_ms(img.id) query = """\ SELECT r.id ,r.mon_src ,rf.f_datapoints FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset_id)s AND rf.runcat = r.id ORDER BY r.wm_ra ,r.mon_src """ cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id}) runcat_flux = get_db_rows_as_dicts(cursor) self.assertEqual(len(runcat_flux), 3) # First entry (lowest RA, mon_src = False) is the regular one; self.assertEqual(runcat_flux[0]['mon_src'], False) # The higher RA source is the monitoring one self.assertEqual(runcat_flux[1]['mon_src'], True) self.assertEqual(runcat_flux[2]['mon_src'], True) for entry in runcat_flux: self.assertEqual(entry['f_datapoints'], len(self.im_params)) #Let's verify the association types blind_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[0]['id']) superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[1]['id']) offset_mon_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[2]['id']) assoc_lists = [blind_src_assocs, superimposed_mon_src_assocs, offset_mon_src_assocs] for al in assoc_lists: self.assertEqual(len(al), 3) # The individual light-curve datapoints for the "normal" source # It was new at first timestep self.assertEqual(blind_src_assocs[0]['type'], 4) self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8) self.assertEqual(offset_mon_src_assocs[0]['type'], 8) for idx, img_pars in enumerate(self.im_params): if idx != 0: self.assertEqual(blind_src_assocs[idx]['type'], 3) self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9) self.assertEqual(offset_mon_src_assocs[idx]['type'], 9) #And the extraction types: self.assertEqual(blind_src_assocs[idx]['extract_type'],0) self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2) self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2) #Sanity check the timestamps while we're at it for al in assoc_lists: self.assertEqual(al[idx]['taustart_ts'], img_pars['taustart_ts'])
def test_basic_case(self): im_params = self.im_params blind_src = db_subs.example_extractedsource_tuple( ra=im_params[0]['centre_ra'], dec=im_params[0]['centre_decl'], ) superimposed_mon_src = blind_src mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001) # Simulate a source that does not get fit, for good measure: mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.) #Sorted by increasing RA: mon_srcs = [ superimposed_mon_src, mon_src_in_field, mon_src_out_of_field ] mon_posns = [(m.ra, m.dec) for m in mon_srcs] dbgen.insert_monitor_positions(self.dataset.id, mon_posns) images = [] for img_pars in self.im_params: img = tkp.db.Image(dataset=self.dataset, data=img_pars) dbgen.insert_extracted_sources(img.id, [blind_src], 'blind') associate_extracted_sources(img.id, deRuiter_r=5.68) nd_requests = get_nulldetections(img.id) self.assertEqual(len(nd_requests), 0) mon_requests = dbmon.get_monitor_entries(self.dataset.id) self.assertEqual(len(mon_requests), len(mon_srcs)) # mon requests is a list of tuples [(id,ra,decl)] # Ensure sorted by RA for cross-checking: mon_requests = sorted(mon_requests, key=lambda s: s[1]) for idx in range(len(mon_srcs)): self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra) self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec) #Insert fits for the in-field sources and then associate dbgen.insert_extracted_sources( img.id, [superimposed_mon_src, mon_src_in_field], 'ff_ms', ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]]) dbmon.associate_ms(img.id) query = """\ SELECT r.id ,r.mon_src ,rf.f_datapoints FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset_id)s AND rf.runcat = r.id ORDER BY r.wm_ra ,r.mon_src """ cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id}) runcat_flux = get_db_rows_as_dicts(cursor) self.assertEqual(len(runcat_flux), 3) # First entry (lowest RA, mon_src = False) is the regular one; self.assertEqual(runcat_flux[0]['mon_src'], False) # The higher RA source is the monitoring one self.assertEqual(runcat_flux[1]['mon_src'], True) self.assertEqual(runcat_flux[2]['mon_src'], True) for entry in runcat_flux: self.assertEqual(entry['f_datapoints'], len(self.im_params)) #Let's verify the association types blind_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[0]['id']) superimposed_mon_src_assocs = get_assoc_entries( self.dataset.database, runcat_flux[1]['id']) offset_mon_src_assocs = get_assoc_entries(self.dataset.database, runcat_flux[2]['id']) assoc_lists = [ blind_src_assocs, superimposed_mon_src_assocs, offset_mon_src_assocs ] for al in assoc_lists: self.assertEqual(len(al), 3) # The individual light-curve datapoints for the "normal" source # It was new at first timestep self.assertEqual(blind_src_assocs[0]['type'], 4) self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8) self.assertEqual(offset_mon_src_assocs[0]['type'], 8) for idx, img_pars in enumerate(self.im_params): if idx != 0: self.assertEqual(blind_src_assocs[idx]['type'], 3) self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9) self.assertEqual(offset_mon_src_assocs[idx]['type'], 9) #And the extraction types: self.assertEqual(blind_src_assocs[idx]['extract_type'], 0) self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'], 2) self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2) #Sanity check the timestamps while we're at it for al in assoc_lists: self.assertEqual(al[idx]['taustart_ts'], img_pars['taustart_ts'])
def test_many2manyflux_reduced_to_two_1to1(self): """ (See also assoc. test test_many2many_reduced_to_two_1to1 ) In this test-case we cross-associate between a rhombus of sources spread about a central position, east-west in the first image, north-south in the second. The latter, north-south pair are slightly offset towards positive RA and negative RA respectively. The result is that the candidate associations are pruned down to two one-to-one pairings.. """ dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: n-m, ' + self._testMethodName}) n_images = 2 im_params = db_subs.generate_timespaced_dbimages_data(n_images) centre_ra, centre_dec = 123., 10.5, offset_deg = 20 / 3600. #20 arcsec tiny_offset_deg = 1 / 3600. #1 arcsec eastern_src = db_subs.example_extractedsource_tuple( ra=centre_ra + offset_deg, dec=centre_dec, peak = 1.5, peak_err = 1e-1, flux = 3.0, flux_err = 1e-1,) western_src = db_subs.example_extractedsource_tuple( ra=centre_ra - offset_deg, dec=centre_dec, peak = 1.7, peak_err = 1e-1, flux = 3.2, flux_err = 1e-1,) northern_source = db_subs.example_extractedsource_tuple( ra=centre_ra + tiny_offset_deg, dec=centre_dec + offset_deg, peak = 1.8, peak_err = 1e-1, flux = 3.3, flux_err = 1e-1, ) southern_source = db_subs.example_extractedsource_tuple( ra=centre_ra - tiny_offset_deg, dec=centre_dec - offset_deg, peak = 1.4, peak_err = 1e-1, flux = 2.9, flux_err = 1e-1,) # image 1 image1 = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0]) dbgen.insert_extracted_sources( image1.id, [eastern_src,western_src], 'blind') associate_extracted_sources(image1.id, deRuiter_r = 3.717) # image 2 image2 = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1]) dbgen.insert_extracted_sources( image2.id, [northern_source, southern_source], 'blind') associate_extracted_sources(image2.id, deRuiter_r = 3.717) # Manually compose the lists of sources we expect to see associated # into runningcatalog entries: # NB img1_srclist[1] has larger RA value. lightcurves_sorted_by_ra =[] lightcurves_sorted_by_ra.append( [western_src, southern_source]) lightcurves_sorted_by_ra.append( [eastern_src, northern_source]) #Check the summary statistics (avg flux, etc) query = """\ SELECT rf.avg_f_int ,rf.avg_f_int_sq ,avg_weighted_f_int ,avg_f_int_weight FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset)s AND r.id = rf.runcat ORDER BY r.wm_ra, r.wm_decl """ self.database.cursor.execute(query, {'dataset': dataset.id}) runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra)) for idx, flux_summary in enumerate(runcat_flux_entries): py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx]) for key in flux_summary.keys(): self.assertAlmostEqual(flux_summary[key], py_results[-1][key]) #Now check the per-timestep statistics (variability indices) sorted_runcat_ids = columns_from_table('runningcatalog', where={'dataset':dataset.id}, order='wm_ra,wm_decl') sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids] for idx, rcid in enumerate(sorted_runcat_ids): db_indices = db_queries.get_assoc_entries(self.database, rcid) py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx]) self.assertEqual(len(db_indices), len(py_indices)) for nstep in range(len(db_indices)): for key in ('v_int', 'eta_int', 'f_datapoints'): self.assertAlmostEqual(db_indices[nstep][key], py_indices[nstep][key])
def test_one2manyflux(self): dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: 1-n'}) n_images = 2 im_params = db_subs.generate_timespaced_dbimages_data(n_images) central_ra, central_dec = 123.1235, 10.55, position_offset_deg = 100./3600 #100 arcsec = 0.03 deg approx # image 1 image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0]) imageid1 = image.id img1_srclist = [] # 1 source img1_srclist.append(db_subs.example_extractedsource_tuple(central_ra, central_dec, peak = 1.5, peak_err = 5e-1, flux = 3.0, flux_err = 5e-1, )) dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind') associate_extracted_sources(imageid1, deRuiter_r=3.717) # image 2 image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1]) imageid2 = image.id img2_srclist = [] # 2 sources (both close to source 1, catching the 1-to-many case) img2_srclist.append(db_subs.example_extractedsource_tuple( central_ra, central_dec, peak = 1.6, peak_err = 5e-1, flux = 3.2, flux_err = 5e-1, )) img2_srclist.append(db_subs.example_extractedsource_tuple( central_ra + position_offset_deg, central_dec, peak = 1.9, peak_err = 5e-1, flux = 3.4, flux_err = 5e-1, )) dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind') associate_extracted_sources(imageid2, deRuiter_r=3.717) # Manually compose the lists of sources we expect to see associated # into runningcatalog entries: # NB img2_srclist[1] has larger RA value. lightcurves_sorted_by_ra =[] lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[0]]) lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[1]]) #Check the summary statistics (avg flux, etc) query = """\ SELECT rf.avg_f_int ,rf.avg_f_int_sq ,avg_weighted_f_int ,avg_f_int_weight FROM runningcatalog r ,runningcatalog_flux rf WHERE r.dataset = %(dataset)s AND r.id = rf.runcat ORDER BY r.wm_ra """ self.database.cursor.execute(query, {'dataset': dataset.id}) runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor) self.assertEqual(len(runcat_flux_entries), 2) for idx, flux_summary in enumerate(runcat_flux_entries): py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx]) for key in flux_summary.keys(): self.assertAlmostEqual(flux_summary[key], py_results[-1][key]) #Now check the per-timestep statistics (variability indices) sorted_runcat_ids = columns_from_table('runningcatalog', where={'dataset':dataset.id}, order='wm_ra') sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids] for idx, rcid in enumerate(sorted_runcat_ids): db_indices = db_queries.get_assoc_entries(self.database, rcid) py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx]) self.assertEqual(len(db_indices), len(py_indices)) for nstep in range(len(db_indices)): for key in ('v_int', 'eta_int', 'f_datapoints'): self.assertAlmostEqual(db_indices[nstep][key], py_indices[nstep][key])