def test_full_transient_search_routine(self):
bands = self.dataset.frequency_bands()
self.assertEqual(len(bands), 1)
# First run with lax limits:
transients = multi_epoch_transient_search(
eta_lim=1.1, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1
)
self.assertEqual(len(transients), 3)
qry = """\
SELECT tr.*
FROM transient tr
,runningcatalog rc
WHERE rc.dataset = %(dsid)s
AND tr.runcat = rc.id
"""
cursor = self.database.connection.cursor()
cursor.execute(qry, {"dsid": self.dataset.id})
transient_table_entries = get_db_rows_as_dicts(cursor)
self.assertEqual(len(transient_table_entries), len(transients))
# for t in all_transients:
# print "V_int:", t['v_int'], " eta_int:", t['eta_int']
# Now test thresholding:
more_highly_variable = sum(t["v_int"] > 2.0 for t in transients)
very_non_flat = sum(t["eta_int"] > 100.0 for t in transients)
transients = multi_epoch_transient_search(
eta_lim=1.1, V_lim=2.0, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1
)
self.assertEqual(len(transients), more_highly_variable)
transients = multi_epoch_transient_search(
eta_lim=100, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1
)
self.assertEqual(len(transients), very_non_flat)
def search_transients(image_id, parset):
logger.info("Finding transient sources...")
transients = multi_epoch_transient_search(image_id=image_id,
eta_lim=parset['eta_lim'],
V_lim=parset['v_lim'],
probability_threshold=parset['threshold'],
minpoints=parset['minpoints'])
return transients
def test_single_band_transient_search(self):
"""test_single_band_transient_search
Test simplest functional case - only source is a transient source.
This makes it simple to verify the database properties.
"""
# We have to add a dataset, some images all with some measurements
# After insertion, and source association, we run the transient search.
dataset = tkp.db.DataSet(database=self.database,
data={'description':"Trans:"
+ self._testMethodName})
n_images = 4
im_params = db_subs.example_dbimage_datasets(n_images)
TransientSource = db_subs.MockSource(
ex=db_subs.example_extractedsource_tuple(),
lightcurve=[
db_subs.MockLCPoint(index=2, peak=20e-2, flux=20e-2, sigma=200),
db_subs.MockLCPoint(index=3, peak=1e-2, flux=1e-2, sigma=10),
]
)
measurements = TransientSource.synthesise_measurements(
n_images,
include_non_detections=True)
images = []
for idx in range(len(im_params)):
image = tkp.db.Image(dataset=dataset, data=im_params[idx])
images.append(image)
if measurements[idx] is not None:
image.insert_extracted_sources([ measurements[idx] ])
image.associate_extracted_sources(deRuiter_r=3.7)
freq_bands = dataset.frequency_bands()
self.assertEqual(len(freq_bands), 1)
transients = multi_epoch_transient_search(
eta_lim=1,
V_lim=0.1,
probability_threshold=0.7,
minpoints=1,
image_id=image.id)
# Check the number of detected transients
self.assertEqual(len(transients), 1)
# Check that the bands for the images are the same as the transient's band
freq_bands = dataset.frequency_bands()
self.assertEqual(len(freq_bands), 1)
for tr in transients:
self.assertEqual(freq_bands[0], tr['band'])
runcats = dataset.runcat_entries()
self.assertEqual(len(runcats), 1)
for tr in transients:
self.assertEqual(runcats[0]['runcat'], tr['runcat'])
# Check that the trigger xtrsrc happened in the third image
query = """\
select taustart_ts
from extractedsource x
,image i
where x.image = i.id
and x.id = (select trigger_xtrsrc
from transient t
,runningcatalog r
where t.runcat = r.id
and r.dataset = %s
)
"""
self.database.cursor.execute(query, (dataset.id,))
taustart_ts = zip(*self.database.cursor.fetchall())[0]
self.assertEqual(len(taustart_ts), 1)
ts = taustart_ts[0]
self.assertEqual(ts, images[2].taustart_ts)
# Check the variability indices, first eta_int:
#query = """\
#select sum((f_int - 0.06) * (f_int-0.06) / (f_int_err * f_int_err)) / 3
# from assocxtrsource a
# ,extractedsource x
# where a.xtrsrc = x.id
# and a.runcat = 7
#"""
#FIXME: I'm not sure what this query is for?
query = """\
def test_intermittentToMonitorlist(self):
dataset = tkp.db.DataSet(database=self.database, data={'description': "Monlist:" + self._testMethodName})
n_images = 3
im_params = db_subs.example_dbimage_datasets(n_images)
steady_srcs = []
# We will work with 2 sources per image
# one being detected in all images and not in the monlist
# the second having a null-detection in the second image
# and stored in the monlist
n_steady_srcs = 2
for i in range(n_steady_srcs):
src = db_subs.example_extractedsource_tuple()
src = src._replace(ra=src.ra + 2 * i)
steady_srcs.append(src)
for idx, im in enumerate(im_params):
image = tkp.db.Image(database=self.database, dataset=dataset, data=im)
if idx == 1:
# The second image has a null detection, so only the first source is detected
image.insert_extracted_sources(steady_srcs[0:1])
else:
image.insert_extracted_sources(steady_srcs)
# First, we check for null detections
nd = monitoringlist.get_nulldetections(image.id, deRuiter_r=3.717)
if idx == 0:
self.assertEqual(len(nd), 0)
elif idx == 1:
self.assertEqual(len(nd), 1)
# The null detection is found,
# We simulate the forced fit result back into extractedsource
# Check that the null-detection ra is the ra of source two
self.assertEqual(nd[0][0], steady_srcs[1].ra)
#print "nd=",nd
tuple_ff_nd = steady_srcs[1:2]
monitoringlist.insert_forcedfits_into_extractedsource(image.id, tuple_ff_nd, 'ff_nd')
elif idx == 2:
self.assertEqual(len(nd), 0)
# Secondly, we do the source association
dbass.associate_extracted_sources(image.id, deRuiter_r=3.717)
monitoringlist.add_nulldetections(image.id)
# We also need to run the transient search in order to pick up the variable
# eta_lim, V_lim, prob_threshold, minpoints, resp.
transients = tr_search.multi_epoch_transient_search(image.id,
0.0,
0.0,
0.5,
1)
# Adjust (insert/update/remove) transients in monlist as well
monitoringlist.adjust_transients_in_monitoringlist(image.id,
transients)
# So after the three images have been processed,
# We should have the null-detection source in the monlist
# Get the null detection in extractedsource
# These are of extract_type = 1
query = """\
select x.id
from extractedsource x
,image i
where x.image = i.id
and i.dataset = %s
and x.extract_type = 1
"""
self.database.cursor.execute(query, (dataset.id,))
result = zip(*self.database.cursor.fetchall())
null_det = result[0]
self.assertEqual(len(null_det), 1)
query = """\
select a.runcat
,a.xtrsrc
,r.wm_ra
,r.wm_decl
from assocxtrsource a
,extractedsource x
,image i
,runningcatalog r
where a.xtrsrc = x.id
and x.id = %s
and x.image = i.id
and i.dataset = %s
and a.runcat = r.id
and r.dataset = i.dataset
"""
self.database.cursor.execute(query, (null_det[0], dataset.id,))
result = zip(*self.database.cursor.fetchall())
assocruncat = result[0]
xtrsrc = result[1]
wm_ra = result[2]
wm_decl = result[3]
self.assertEqual(len(assocruncat), 1)
query = """\
SELECT runcat
,ra
,decl
FROM monitoringlist
WHERE dataset = %s
"""
self.database.cursor.execute(query, (dataset.id,))
result = zip(*self.database.cursor.fetchall())
# print "len(result)=",len(result)
self.assertEqual(len(result), 3)
#self.assertEqual(0, 1)
monruncat = result[0]
ra = result[1]
decl = result[2]
self.assertEqual(len(monruncat), 1)
self.assertEqual(monruncat[0], assocruncat[0])
self.assertEqual(ra[0], wm_ra[0])
self.assertAlmostEqual(decl[0], wm_decl[0])
