def test_full_transient_search_routine(self): bands = self.dataset.frequency_bands() self.assertEqual(len(bands), 1) # First run with lax limits: transients = multi_epoch_transient_search( eta_lim=1.1, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1 ) self.assertEqual(len(transients), 3) qry = """\ SELECT tr.* FROM transient tr ,runningcatalog rc WHERE rc.dataset = %(dsid)s AND tr.runcat = rc.id """ cursor = self.database.connection.cursor() cursor.execute(qry, {"dsid": self.dataset.id}) transient_table_entries = get_db_rows_as_dicts(cursor) self.assertEqual(len(transient_table_entries), len(transients)) # for t in all_transients: # print "V_int:", t['v_int'], " eta_int:", t['eta_int'] # Now test thresholding: more_highly_variable = sum(t["v_int"] > 2.0 for t in transients) very_non_flat = sum(t["eta_int"] > 100.0 for t in transients) transients = multi_epoch_transient_search( eta_lim=1.1, V_lim=2.0, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1 ) self.assertEqual(len(transients), more_highly_variable) transients = multi_epoch_transient_search( eta_lim=100, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1 ) self.assertEqual(len(transients), very_non_flat)
def search_transients(image_id, parset): logger.info("Finding transient sources...") transients = multi_epoch_transient_search(image_id=image_id, eta_lim=parset['eta_lim'], V_lim=parset['v_lim'], probability_threshold=parset['threshold'], minpoints=parset['minpoints']) return transients
def test_single_band_transient_search(self): """test_single_band_transient_search Test simplest functional case - only source is a transient source. This makes it simple to verify the database properties. """ # We have to add a dataset, some images all with some measurements # After insertion, and source association, we run the transient search. dataset = tkp.db.DataSet(database=self.database, data={'description':"Trans:" + self._testMethodName}) n_images = 4 im_params = db_subs.example_dbimage_datasets(n_images) TransientSource = db_subs.MockSource( ex=db_subs.example_extractedsource_tuple(), lightcurve=[ db_subs.MockLCPoint(index=2, peak=20e-2, flux=20e-2, sigma=200), db_subs.MockLCPoint(index=3, peak=1e-2, flux=1e-2, sigma=10), ] ) measurements = TransientSource.synthesise_measurements( n_images, include_non_detections=True) images = [] for idx in range(len(im_params)): image = tkp.db.Image(dataset=dataset, data=im_params[idx]) images.append(image) if measurements[idx] is not None: image.insert_extracted_sources([ measurements[idx] ]) image.associate_extracted_sources(deRuiter_r=3.7) freq_bands = dataset.frequency_bands() self.assertEqual(len(freq_bands), 1) transients = multi_epoch_transient_search( eta_lim=1, V_lim=0.1, probability_threshold=0.7, minpoints=1, image_id=image.id) # Check the number of detected transients self.assertEqual(len(transients), 1) # Check that the bands for the images are the same as the transient's band freq_bands = dataset.frequency_bands() self.assertEqual(len(freq_bands), 1) for tr in transients: self.assertEqual(freq_bands[0], tr['band']) runcats = dataset.runcat_entries() self.assertEqual(len(runcats), 1) for tr in transients: self.assertEqual(runcats[0]['runcat'], tr['runcat']) # Check that the trigger xtrsrc happened in the third image query = """\ select taustart_ts from extractedsource x ,image i where x.image = i.id and x.id = (select trigger_xtrsrc from transient t ,runningcatalog r where t.runcat = r.id and r.dataset = %s ) """ self.database.cursor.execute(query, (dataset.id,)) taustart_ts = zip(*self.database.cursor.fetchall())[0] self.assertEqual(len(taustart_ts), 1) ts = taustart_ts[0] self.assertEqual(ts, images[2].taustart_ts) # Check the variability indices, first eta_int: #query = """\ #select sum((f_int - 0.06) * (f_int-0.06) / (f_int_err * f_int_err)) / 3 # from assocxtrsource a # ,extractedsource x # where a.xtrsrc = x.id # and a.runcat = 7 #""" #FIXME: I'm not sure what this query is for? query = """\
def test_intermittentToMonitorlist(self): dataset = tkp.db.DataSet(database=self.database, data={'description': "Monlist:" + self._testMethodName}) n_images = 3 im_params = db_subs.example_dbimage_datasets(n_images) steady_srcs = [] # We will work with 2 sources per image # one being detected in all images and not in the monlist # the second having a null-detection in the second image # and stored in the monlist n_steady_srcs = 2 for i in range(n_steady_srcs): src = db_subs.example_extractedsource_tuple() src = src._replace(ra=src.ra + 2 * i) steady_srcs.append(src) for idx, im in enumerate(im_params): image = tkp.db.Image(database=self.database, dataset=dataset, data=im) if idx == 1: # The second image has a null detection, so only the first source is detected image.insert_extracted_sources(steady_srcs[0:1]) else: image.insert_extracted_sources(steady_srcs) # First, we check for null detections nd = monitoringlist.get_nulldetections(image.id, deRuiter_r=3.717) if idx == 0: self.assertEqual(len(nd), 0) elif idx == 1: self.assertEqual(len(nd), 1) # The null detection is found, # We simulate the forced fit result back into extractedsource # Check that the null-detection ra is the ra of source two self.assertEqual(nd[0][0], steady_srcs[1].ra) #print "nd=",nd tuple_ff_nd = steady_srcs[1:2] monitoringlist.insert_forcedfits_into_extractedsource(image.id, tuple_ff_nd, 'ff_nd') elif idx == 2: self.assertEqual(len(nd), 0) # Secondly, we do the source association dbass.associate_extracted_sources(image.id, deRuiter_r=3.717) monitoringlist.add_nulldetections(image.id) # We also need to run the transient search in order to pick up the variable # eta_lim, V_lim, prob_threshold, minpoints, resp. transients = tr_search.multi_epoch_transient_search(image.id, 0.0, 0.0, 0.5, 1) # Adjust (insert/update/remove) transients in monlist as well monitoringlist.adjust_transients_in_monitoringlist(image.id, transients) # So after the three images have been processed, # We should have the null-detection source in the monlist # Get the null detection in extractedsource # These are of extract_type = 1 query = """\ select x.id from extractedsource x ,image i where x.image = i.id and i.dataset = %s and x.extract_type = 1 """ self.database.cursor.execute(query, (dataset.id,)) result = zip(*self.database.cursor.fetchall()) null_det = result[0] self.assertEqual(len(null_det), 1) query = """\ select a.runcat ,a.xtrsrc ,r.wm_ra ,r.wm_decl from assocxtrsource a ,extractedsource x ,image i ,runningcatalog r where a.xtrsrc = x.id and x.id = %s and x.image = i.id and i.dataset = %s and a.runcat = r.id and r.dataset = i.dataset """ self.database.cursor.execute(query, (null_det[0], dataset.id,)) result = zip(*self.database.cursor.fetchall()) assocruncat = result[0] xtrsrc = result[1] wm_ra = result[2] wm_decl = result[3] self.assertEqual(len(assocruncat), 1) query = """\ SELECT runcat ,ra ,decl FROM monitoringlist WHERE dataset = %s """ self.database.cursor.execute(query, (dataset.id,)) result = zip(*self.database.cursor.fetchall()) # print "len(result)=",len(result) self.assertEqual(len(result), 3) #self.assertEqual(0, 1) monruncat = result[0] ra = result[1] decl = result[2] self.assertEqual(len(monruncat), 1) self.assertEqual(monruncat[0], assocruncat[0]) self.assertEqual(ra[0], wm_ra[0]) self.assertAlmostEqual(decl[0], wm_decl[0])