def setUpClass(cls):
#NB This is basically nonsense, we're just selecting any old
#extracted source so we can check the feature extraction syntax.
xtrsrc_qry = """SELECT id as trigger_xtrsrc
FROM extractedsource"""
cursor = execute(xtrsrc_qry)
cls.transients = get_db_rows_as_dicts(cursor)
def get_assoc_entries(db, runcat_id):
"""
Return the full history of variability indices for a runcat entry,
ordered by time.
"""
query = """\
select a.runcat
,a.xtrsrc
,x.extract_type
,i.taustart_ts
,a.v_int
,a.eta_int
,a.f_datapoints
,a.type
,r.mon_src
from assocxtrsource a
,extractedsource x
,image i
,runningcatalog r
where a.xtrsrc = x.id
and x.image = i.id
and a.runcat = r.id
and r.id = %(runcat_id)s
order by i.taustart_ts
"""
db.cursor.execute(query, {'runcat_id': runcat_id})
return get_db_rows_as_dicts(db.cursor)
def get_sources_filtered_by_final_variability(dataset_id,
eta_min,
v_min,
# minpoints
):
"""
Search the database to find high-variability lightcurves.
Uses the variability associated with the last datapoint in a lightcurve
as the key criteria.
Args:
dataset_id (int): Dataset to search
eta_min (float): Minimum value of eta-index to return.
v_min (float): Minimum value of V-index to return.
Returns:
list: (list of dicts) Each dict represents a runningcatalog_flux entry
matching the filter criteria.
"""
query = """\
SELECT rc.id as runcat_id
,image.band
,ax.v_int
,ax.eta_int
FROM runningcatalog as rc
JOIN assocxtrsource as ax ON ax.runcat = rc.id
JOIN extractedsource as ex ON ax.xtrsrc = ex.id
JOIN image ON ex.image = image.id
JOIN (
-- Determine which are the most recent variability values
-- for each lightcurve.
SELECT
a.runcat as runcat_id,
i.band as band,
max(i.taustart_ts) as MaxTimestamp
FROM
assocxtrsource a
JOIN extractedsource e ON a.xtrsrc = e.id
JOIN image i ON e.image = i.id
GROUP BY
runcat_id, band
) last_timestamps
ON rc.id = last_timestamps.runcat_id
AND image.band = last_timestamps.band
AND image.taustart_ts = last_timestamps.MaxTimestamp
WHERE rc.dataset = %(dataset_id)s
AND eta_int >= %(eta_min)s
AND v_int >= %(v_min)s
"""
cursor = tkp.db.Database().cursor
cursor.execute(query, {'dataset_id': dataset_id,
'eta_min':eta_min,
'v_min':v_min,
})
transients = get_db_rows_as_dicts(cursor)
return transients
def get_assoc_entries(db, runcat_id):
"""
Return the full history of variability indices for a runcat entry,
ordered by time.
"""
query = """\
select a.runcat
,a.xtrsrc
,x.extract_type
,i.taustart_ts
,a.v_int
,a.eta_int
,a.f_datapoints
,a.type
,r.mon_src
from assocxtrsource a
,extractedsource x
,image i
,runningcatalog r
where a.xtrsrc = x.id
and x.image = i.id
and a.runcat = r.id
and r.id = %(runcat_id)s
order by i.taustart_ts
"""
db.cursor.execute(query, {'runcat_id': runcat_id})
return get_db_rows_as_dicts(db.cursor)
def get_sources_filtered_by_final_variability(dataset_id,
eta_min,
v_min,
# minpoints
):
"""
Search the database to find high-variability lightcurves.
Uses the variability associated with the last datapoint in a lightcurve
as the key criteria.
Args:
dataset_id (int): Dataset to search
eta_min (float): Minimum value of eta-index to return.
v_min (float): Minimum value of V-index to return.
Returns:
list: (list of dicts) Each dict represents a runningcatalog_flux entry
matching the filter criteria.
"""
query = """\
SELECT rc.id as runcat_id
,image.band
,ax.v_int
,ax.eta_int
FROM runningcatalog as rc
JOIN assocxtrsource as ax ON ax.runcat = rc.id
JOIN extractedsource as ex ON ax.xtrsrc = ex.id
JOIN image ON ex.image = image.id
JOIN (
-- Determine which are the most recent variability values
-- for each lightcurve.
SELECT
a.runcat as runcat_id,
i.band as band,
max(i.taustart_ts) as MaxTimestamp
FROM
assocxtrsource a
JOIN extractedsource e ON a.xtrsrc = e.id
JOIN image i ON e.image = i.id
GROUP BY
runcat_id, band
) last_timestamps
ON rc.id = last_timestamps.runcat_id
AND image.band = last_timestamps.band
AND image.taustart_ts = last_timestamps.MaxTimestamp
WHERE rc.dataset = %(dataset_id)s
AND eta_int >= %(eta_min)s
AND v_int >= %(v_min)s
"""
cursor = tkp.db.Database().connection.cursor()
cursor.execute(query, {'dataset_id': dataset_id,
'eta_min':eta_min,
'v_min':v_min,
})
transients = get_db_rows_as_dicts(cursor)
return transients
def test_full_transient_search_routine(self):
bands = self.dataset.frequency_bands()
self.assertEqual(len(bands), 1)
# First run with lax limits:
transients = multi_epoch_transient_search(
eta_lim=1.1, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1
)
self.assertEqual(len(transients), 3)
qry = """\
SELECT tr.*
FROM transient tr
,runningcatalog rc
WHERE rc.dataset = %(dsid)s
AND tr.runcat = rc.id
"""
cursor = self.database.connection.cursor()
cursor.execute(qry, {"dsid": self.dataset.id})
transient_table_entries = get_db_rows_as_dicts(cursor)
self.assertEqual(len(transient_table_entries), len(transients))
# for t in all_transients:
# print "V_int:", t['v_int'], " eta_int:", t['eta_int']
# Now test thresholding:
more_highly_variable = sum(t["v_int"] > 2.0 for t in transients)
very_non_flat = sum(t["eta_int"] > 100.0 for t in transients)
transients = multi_epoch_transient_search(
eta_lim=1.1, V_lim=2.0, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1
)
self.assertEqual(len(transients), more_highly_variable)
transients = multi_epoch_transient_search(
eta_lim=100, V_lim=0.01, probability_threshold=0.01, image_id=self.db_imgs[-1].id, minpoints=1
)
self.assertEqual(len(transients), very_non_flat)
def test_two_field_overlap_new_transient(self):
"""Now for something more interesting - two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at 2nd timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.example_dbimage_datasets(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
imgs[0].insert_extracted_sources([lower_steady_src, overlap_steady_src])
nd_posns = dbmon.get_nulldetections(imgs[0].id, deRuiter_r=1)
self.assertEqual(len(nd_posns), 0)
imgs[0].associate_extracted_sources(deRuiter_r=0.1)
imgs[1].insert_extracted_sources([upper_steady_src, overlap_steady_src,
overlap_transient])
nd_posns = dbmon.get_nulldetections(imgs[1].id, deRuiter_r=1)
self.assertEqual(len(nd_posns), 0)
imgs[1].associate_extracted_sources(deRuiter_r=0.1)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
monlist = columns_from_table('monitoringlist',
where={'dataset': self.dataset.id})
self.assertEqual(len(monlist), 1)
transients_qry = """\
SELECT *
FROM transient tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(transients_qry, (self.dataset.id,))
transients = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(transients), 1)
def test_two_field_overlap_new_transient(self):
"""Now for something more interesting - two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at 2nd timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
imgs[0].insert_extracted_sources(
[lower_steady_src, overlap_steady_src])
imgs[0].associate_extracted_sources(
deRuiter_r=0.1, new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
imgs[1].insert_extracted_sources(
[upper_steady_src, overlap_steady_src, overlap_transient])
imgs[1].associate_extracted_sources(
deRuiter_r=0.1, new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 0)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id, ))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 1)
def get_newsources_for_dataset(dsid):
"""
Returns dicts representing all newsources for this dataset.
Args:
dsid: Dataset id
Returns:
list: (list of dicts) Each dict represents one newsource.
The dict keys are all the columns in the newsources table, plus
the 'taustart_ts' from the image table, which represents the
trigger time.
"""
qry = """\
SELECT tr.id
,tr.previous_limits_image
,rc.id as runcat_id
,img.taustart_ts
,img.band
,ax.v_int
,ax.eta_int
, ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_min)
/ limits_image.rms_min) AS low_thresh_sigma
, ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_max)
/ limits_image.rms_max) AS high_thresh_sigma
FROM newsource tr
,runningcatalog rc
,extractedsource ex
,image img
,assocxtrsource ax
,image limits_image
WHERE rc.dataset = %(dsid)s
AND tr.runcat = rc.id
AND tr.trigger_xtrsrc = ex.id
AND ex.image = img.id
AND ax.runcat = rc.id
AND ax.xtrsrc = ex.id
AND tr.previous_limits_image = limits_image.id
"""
cursor = Database().cursor
cursor.execute(qry, {'dsid':dsid})
newsource_rows_for_dataset = get_db_rows_as_dicts(cursor)
return newsource_rows_for_dataset
def get_newsources_for_dataset(dsid):
"""
Returns dicts representing all newsources for this dataset.
Args:
dsid: Dataset id
Returns:
list: (list of dicts) Each dict represents one newsource.
The dict keys are all the columns in the newsources table, plus
the 'taustart_ts' from the image table, which represents the
trigger time.
"""
qry = """\
SELECT tr.id
,tr.previous_limits_image
,rc.id as runcat_id
,img.taustart_ts
,img.band
,ax.v_int
,ax.eta_int
, ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_min)
/ limits_image.rms_min) AS low_thresh_sigma
, ((ex.f_peak - limits_image.detection_thresh*limits_image.rms_max)
/ limits_image.rms_max) AS high_thresh_sigma
FROM newsource tr
,runningcatalog rc
,extractedsource ex
,image img
,assocxtrsource ax
,image limits_image
WHERE rc.dataset = %(dsid)s
AND tr.runcat = rc.id
AND tr.trigger_xtrsrc = ex.id
AND ex.image = img.id
AND ax.runcat = rc.id
AND ax.xtrsrc = ex.id
AND tr.previous_limits_image = limits_image.id
"""
cursor = Database().connection.cursor()
cursor.execute(qry, {'dsid':dsid})
newsource_rows_for_dataset = get_db_rows_as_dicts(cursor)
return newsource_rows_for_dataset
def test_two_field_basic_case(self):
"""
Here we create 2 disjoint image fields, with one source at centre of
each, and check that the second source inserted does not get flagged as
newsource.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 2 + 0.5
imgs = []
for idx in range(len(im_params)):
imgs.append(tkp.db.Image(dataset=self.dataset,
data=im_params[idx]))
for idx in range(len(im_params)):
central_src = db_subs.example_extractedsource_tuple(
ra=im_params[idx]['centre_ra'],
dec=im_params[idx]['centre_decl'])
imgs.append(tkp.db.Image(dataset=self.dataset,
data=im_params[idx]))
imgs[idx].insert_extracted_sources([central_src])
imgs[idx].associate_extracted_sources(deRuiter_r,
new_source_sigma_margin)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 2) #Just a sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id, ))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 0)
def per_timestep_variability_indices(db, runcat_id):
query = """\
select a.runcat
,a.xtrsrc
,i.taustart_ts
,a.v_int
,a.eta_int
from assocxtrsource a
,extractedsource x
,image i
,runningcatalog r
where a.xtrsrc = x.id
and x.image = i.id
and a.runcat = r.id
and r.id = %(runcat_id)s
order by r.wm_ra
,i.taustart_ts
"""
db.cursor.execute(query, {'runcat_id': runcat_id})
return get_db_rows_as_dicts(db.cursor)
def test_two_field_basic_case(self):
"""
Here we create 2 disjoint image fields, with one source at centre of
each, and check that the second source inserted does not get flagged as
newsource.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 2 + 0.5
imgs = []
for idx in range(len(im_params)):
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
for idx in range(len(im_params)):
central_src = db_subs.example_extractedsource_tuple(
ra=im_params[idx]['centre_ra'],
dec=im_params[idx]['centre_decl'])
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
imgs[idx].insert_extracted_sources([central_src])
imgs[idx].associate_extracted_sources(deRuiter_r, new_source_sigma_margin)
runcats = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id})
self.assertEqual(len(runcats), 2) #Just a sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id,))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 0)
def test_lightcurve(self):
# make 4 images with different date
images = []
image_datasets = db_subs.generate_timespaced_dbimages_data(n_images=4,
taustart_ts= datetime.datetime(2010, 3, 3)
)
for dset in image_datasets:
image = Image(dataset=self.dataset, data=dset)
images.append(image)
# 3 sources per image, with different coordinates & flux
data_list = []
for i in range(1, 4):
data_list.append({
'ra': 111.11 + i,
'decl': 11.11 + i,
'i_peak': 10. * i ,
'i_peak_err': 0.1,
})
# Insert the 3 sources in each image, while further varying the flux
lightcurves_sorted_by_ra = [[],[],[]]
for im_idx, image in enumerate(images):
# Create the "source finding results"
# Note that we reuse 'i_peak' as both peak & integrated flux.
img_sources = []
for src_idx, data in enumerate(data_list):
src = db_subs.example_extractedsource_tuple(
ra = data['ra'],dec=data['decl'],
peak=data['i_peak']* (1 + im_idx),
flux = data['i_peak']* (1 + im_idx)
)
lightcurves_sorted_by_ra[src_idx].append(src)
img_sources.append(src)
insert_extracted_sources(image._id, img_sources)
associate_extracted_sources(image._id, deRuiter_r=3.7,
new_source_sigma_margin=3)
# updates the dataset and its set of images
self.dataset.update()
self.dataset.update_images()
# update the images and their sets of sources
for image in self.dataset.images:
image.update()
image.update_sources()
# Now pick last image, select the first source (smallest RA)
# and extract its light curve
sources = self.dataset.images[-1].sources
sources = sorted(sources, key=attrgetter('ra'))
lightcurve = ligtcurve_func(sources[0]._id)
# check if the sources are associated in all images
self.assertEqual(len(images), len(lightcurve))
self.assertEqual(lightcurve[0][0], datetime.datetime(2010, 3, 3, 0, 0))
self.assertEqual(lightcurve[1][0], datetime.datetime(2010, 3, 4, 0, 0))
self.assertEqual(lightcurve[2][0], datetime.datetime(2010, 3, 5, 0, 0))
self.assertEqual(lightcurve[3][0], datetime.datetime(2010, 3, 6, 0, 0))
self.assertAlmostEqual(lightcurve[0][2], 10.)
self.assertAlmostEqual(lightcurve[1][2], 20.)
self.assertAlmostEqual(lightcurve[2][2], 30.)
self.assertAlmostEqual(lightcurve[3][2], 40.)
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': self.dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key],
places=5)
def test_many2manyflux_reduced_to_two_1to1(self):
"""
(See also assoc. test test_many2many_reduced_to_two_1to1 )
In this test-case we cross-associate between a rhombus of sources spread
about a central position, east-west in the first image,
north-south in the second.
The latter, north-south pair are slightly offset towards positive RA
and negative RA respectively.
The result is that the candidate associations are pruned down to
two one-to-one pairings..
"""
dataset = tkp.db.DataSet(database=self.database,
data={
'description':
'flux test set: n-m, ' +
self._testMethodName
})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
centre_ra, centre_dec = 123., 10.5,
offset_deg = 20 / 3600. #20 arcsec
tiny_offset_deg = 1 / 3600. #1 arcsec
eastern_src = db_subs.example_extractedsource_tuple(
ra=centre_ra + offset_deg,
dec=centre_dec,
peak=1.5,
peak_err=1e-1,
flux=3.0,
flux_err=1e-1,
)
western_src = db_subs.example_extractedsource_tuple(
ra=centre_ra - offset_deg,
dec=centre_dec,
peak=1.7,
peak_err=1e-1,
flux=3.2,
flux_err=1e-1,
)
northern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra + tiny_offset_deg,
dec=centre_dec + offset_deg,
peak=1.8,
peak_err=1e-1,
flux=3.3,
flux_err=1e-1,
)
southern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra - tiny_offset_deg,
dec=centre_dec - offset_deg,
peak=1.4,
peak_err=1e-1,
flux=2.9,
flux_err=1e-1,
)
# image 1
image1 = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[0])
dbgen.insert_extracted_sources(image1.id, [eastern_src, western_src],
'blind')
associate_extracted_sources(image1.id, deRuiter_r=3.717)
# image 2
image2 = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[1])
dbgen.insert_extracted_sources(image2.id,
[northern_source, southern_source],
'blind')
associate_extracted_sources(image2.id, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img1_srclist[1] has larger RA value.
lightcurves_sorted_by_ra = []
lightcurves_sorted_by_ra.append([western_src, southern_source])
lightcurves_sorted_by_ra.append([eastern_src, northern_source])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra, r.wm_decl
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries),
len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset': dataset.id},
order='wm_ra,wm_decl')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database, rcid)
py_indices = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_one2manyflux(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-n'})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
central_ra, central_dec = 123.1235, 10.55,
position_offset_deg = 100. / 3600 #100 arcsec = 0.03 deg approx
# image 1
image = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[0])
imageid1 = image.id
img1_srclist = []
# 1 source
img1_srclist.append(
db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak=1.5,
peak_err=5e-1,
flux=3.0,
flux_err=5e-1,
))
dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind')
associate_extracted_sources(imageid1, deRuiter_r=3.717)
# image 2
image = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[1])
imageid2 = image.id
img2_srclist = []
# 2 sources (both close to source 1, catching the 1-to-many case)
img2_srclist.append(
db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak=1.6,
peak_err=5e-1,
flux=3.2,
flux_err=5e-1,
))
img2_srclist.append(
db_subs.example_extractedsource_tuple(
central_ra + position_offset_deg,
central_dec,
peak=1.9,
peak_err=5e-1,
flux=3.4,
flux_err=5e-1,
))
dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind')
associate_extracted_sources(imageid2, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img2_srclist[1] has larger RA value.
lightcurves_sorted_by_ra = []
lightcurves_sorted_by_ra.append([img1_srclist[0], img2_srclist[0]])
lightcurves_sorted_by_ra.append([img1_srclist[0], img2_srclist[1]])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), 2)
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset': dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database, rcid)
py_indices = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def _select_updated_variability_indices(image_id):
"""
Select sources and integrated flux variability indices, for sources which
have had an extra datapoint added by the specified image.
As part of the results we return a field 'new_transient' which is TRUE
if the runcat-source/band combination does not yet have an entry in the
transient table.
NB Variability index = std.dev(flux) / mean(flux).
In the pathological case where mean(flux)==0.0, we simply substitute
variability index = std.dev(flux) / 1e-6 (1 microJansky)
to avoid division by zero errors.
(TO DO: Return indices based on
peak fluxes as well.)
*Args*:
- image_id: the image and thus frequency band being searched for variability
*Returns*:
A list of dicts with keys as follows:
[{ runcat, band, f_datapoints,
wm_ra, wm_decl, wm_ra_err, wm_decl_err,
v_int, eta_int, trigger_xtrsrc, new_transient }]
"""
# Note: We cannot trivially calculate an updated 'siglevel' probability,
# and selecting it from transients gives the *old* value.
# So; we recalculate it later, (using scipy.stats),
# and apply a threshold there.
# NB We also perform a left outer join with the transient table,
# to determine if the source has been inserted into that table yet.
# This allows us to distinguish newly identified transients.
query = """\
SELECT t1.runcat
,t1.band
,t1.f_datapoints
,t1.wm_ra
,t1.wm_decl
,t1.wm_ra_err
,t1.wm_decl_err
,t1.V_int_inter / t1.avg_f_int AS v_int
,t1.eta_int_inter / t1.avg_f_int_weight AS eta_int
,CASE WHEN tr0.trigger_xtrsrc IS NULL
THEN t1.xtrsrc
ELSE tr0.trigger_xtrsrc
END AS trigger_xtrsrc
,CASE WHEN tr0.trigger_xtrsrc IS NULL
THEN TRUE
ELSE FALSE
END AS new_transient
FROM (SELECT rf0.runcat
,rf0.band
,f_datapoints
,wm_ra
,wm_decl
,wm_ra_err
,wm_decl_err
,CASE WHEN avg_f_int = 0.0
THEN 0.000001
ELSE avg_f_int
END AS avg_f_int
,avg_f_int_weight
,CASE WHEN rf0.f_datapoints = 1 THEN 0
WHEN avg_f_int_sq - avg_f_int * avg_f_int < 0 THEN 0
ELSE SQRT(CAST(rf0.f_datapoints AS DOUBLE PRECISION) * (avg_f_int_sq - avg_f_int * avg_f_int)
/ (CAST(rf0.f_datapoints AS DOUBLE PRECISION) - 1.0)
)
END AS V_int_inter
,CASE WHEN rf0.f_datapoints = 1
THEN 0
ELSE (CAST(rf0.f_datapoints AS DOUBLE PRECISION) / (CAST(rf0.f_datapoints AS DOUBLE PRECISION) - 1.0))
* (avg_f_int_weight * avg_weighted_f_int_sq - avg_weighted_f_int * avg_weighted_f_int)
END AS eta_int_inter
,a0.xtrsrc
FROM runningcatalog rc0
,runningcatalog_flux rf0
,image i0
,assocxtrsource a0
,extractedsource x0
WHERE i0.id = %(imgid)s
AND rc0.dataset = i0.dataset
AND rc0.id = rf0.runcat
AND rf0.band = i0.band
AND rc0.id = a0.runcat
AND a0.xtrsrc = x0.id
AND x0.image = %(imgid)s
) t1
LEFT OUTER JOIN transient tr0
ON t1.runcat = tr0.runcat
AND t1.band = tr0.band
ORDER BY t1.runcat
,t1.band
"""
qry_params = {'imgid':image_id}
cursor = tkp.db.execute(query, qry_params)
results = get_db_rows_as_dicts(cursor)
return results
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.)
#Sorted by increasing RA:
mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id,mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests),0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests),len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key = lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(img.id,
[superimposed_mon_src, mon_src_in_field], 'ff_ms',
ff_monitor_ids=[mon_requests[0][0],
mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [blind_src_assocs,
superimposed_mon_src_assocs,
offset_mon_src_assocs]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'],0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.)
#Sorted by increasing RA:
mon_srcs = [
superimposed_mon_src, mon_src_in_field, mon_src_out_of_field
]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id, mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests), 0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests), len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key=lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(
img.id, [superimposed_mon_src, mon_src_in_field],
'ff_ms',
ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(
self.dataset.database, runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [
blind_src_assocs, superimposed_mon_src_assocs,
offset_mon_src_assocs
]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'], 0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],
2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_many2manyflux_reduced_to_two_1to1(self):
"""
(See also assoc. test test_many2many_reduced_to_two_1to1 )
In this test-case we cross-associate between a rhombus of sources spread
about a central position, east-west in the first image,
north-south in the second.
The latter, north-south pair are slightly offset towards positive RA
and negative RA respectively.
The result is that the candidate associations are pruned down to
two one-to-one pairings..
"""
dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: n-m, ' + self._testMethodName})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
centre_ra, centre_dec = 123., 10.5,
offset_deg = 20 / 3600. #20 arcsec
tiny_offset_deg = 1 / 3600. #1 arcsec
eastern_src = db_subs.example_extractedsource_tuple(
ra=centre_ra + offset_deg,
dec=centre_dec,
peak = 1.5, peak_err = 1e-1,
flux = 3.0, flux_err = 1e-1,)
western_src = db_subs.example_extractedsource_tuple(
ra=centre_ra - offset_deg,
dec=centre_dec,
peak = 1.7, peak_err = 1e-1,
flux = 3.2, flux_err = 1e-1,)
northern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra + tiny_offset_deg,
dec=centre_dec + offset_deg,
peak = 1.8, peak_err = 1e-1,
flux = 3.3, flux_err = 1e-1,
)
southern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra - tiny_offset_deg,
dec=centre_dec - offset_deg,
peak = 1.4, peak_err = 1e-1,
flux = 2.9, flux_err = 1e-1,)
# image 1
image1 = tkp.db.Image(database=self.database, dataset=dataset,
data=im_params[0])
dbgen.insert_extracted_sources(
image1.id, [eastern_src,western_src], 'blind')
associate_extracted_sources(image1.id, deRuiter_r = 3.717)
# image 2
image2 = tkp.db.Image(database=self.database, dataset=dataset,
data=im_params[1])
dbgen.insert_extracted_sources(
image2.id, [northern_source, southern_source], 'blind')
associate_extracted_sources(image2.id, deRuiter_r = 3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img1_srclist[1] has larger RA value.
lightcurves_sorted_by_ra =[]
lightcurves_sorted_by_ra.append( [western_src, southern_source])
lightcurves_sorted_by_ra.append( [eastern_src, northern_source])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra, r.wm_decl
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':dataset.id},
order='wm_ra,wm_decl')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_one2manyflux(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-n'})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
central_ra, central_dec = 123.1235, 10.55,
position_offset_deg = 100./3600 #100 arcsec = 0.03 deg approx
# image 1
image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0])
imageid1 = image.id
img1_srclist = []
# 1 source
img1_srclist.append(db_subs.example_extractedsource_tuple(central_ra, central_dec,
peak = 1.5, peak_err = 5e-1,
flux = 3.0, flux_err = 5e-1,
))
dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind')
associate_extracted_sources(imageid1, deRuiter_r=3.717)
# image 2
image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1])
imageid2 = image.id
img2_srclist = []
# 2 sources (both close to source 1, catching the 1-to-many case)
img2_srclist.append(db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak = 1.6, peak_err = 5e-1,
flux = 3.2, flux_err = 5e-1,
))
img2_srclist.append(db_subs.example_extractedsource_tuple(
central_ra + position_offset_deg,
central_dec,
peak = 1.9, peak_err = 5e-1,
flux = 3.4, flux_err = 5e-1,
))
dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind')
associate_extracted_sources(imageid2, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img2_srclist[1] has larger RA value.
lightcurves_sorted_by_ra =[]
lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[0]])
lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[1]])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), 2)
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
