def test_one2oneflux(self):
dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: 1-1'})
n_images = 3
im_params = db_subs.example_dbimage_datasets(n_images)
src_list = []
src = db_subs.example_extractedsource_tuple()
src0 = src._replace(flux=2.0)
src_list.append(src0)
src1 = src._replace(flux=2.5)
src_list.append(src1)
src2 = src._replace(flux=2.4)
src_list.append(src2)
for idx, im in enumerate(im_params):
image = tkp.db.Image(database=self.database, dataset=dataset, data=im)
image.insert_extracted_sources([src_list[idx]])
associate_extracted_sources(image.id, deRuiter_r=3.717)
query = """\
SELECT rf.avg_f_int
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
result = zip(*self.database.cursor.fetchall())
avg_f_int = result[0]
self.assertEqual(len(avg_f_int), 1)
py_metrics = db_subs.lightcurve_metrics(src_list)
self.assertAlmostEqual(avg_f_int[0], py_metrics[-1]['avg_f_int'])
runcat_id = columns_from_table('runningcatalog',
where={'dataset':dataset.id})
self.assertEqual(len(runcat_id),1)
runcat_id = runcat_id[0]['id']
# Check evolution of variability indices
db_metrics = db_queries.per_timestep_variability_indices(self.database,
runcat_id)
self.assertEqual(len(db_metrics), n_images)
# Compare the python- and db-calculated values
for i in range(len(db_metrics)):
for key in ('v_int','eta_int'):
self.assertAlmostEqual(db_metrics[i][key], py_metrics[i][key])
def test_one2manyflux(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-n'})
n_images = 2
im_params = db_subs.example_dbimage_datasets(n_images)
central_ra, central_dec = 123.1235, 10.55,
position_offset_deg = 100./3600 #100 arcsec = 0.03 deg approx
# image 1
image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0])
imageid1 = image.id
img1_srclist = []
# 1 source
img1_srclist.append(db_subs.example_extractedsource_tuple(central_ra, central_dec,
peak = 1.5, peak_err = 5e-1,
flux = 3.0, flux_err = 5e-1,
))
dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind')
associate_extracted_sources(imageid1, deRuiter_r=3.717)
# image 2
image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1])
imageid2 = image.id
img2_srclist = []
# 2 sources (both close to source 1, catching the 1-to-many case)
img2_srclist.append(db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak = 1.6, peak_err = 5e-1,
flux = 3.2, flux_err = 5e-1,
))
img2_srclist.append(db_subs.example_extractedsource_tuple(
central_ra + position_offset_deg,
central_dec,
peak = 1.9, peak_err = 5e-1,
flux = 3.4, flux_err = 5e-1,
))
dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind')
associate_extracted_sources(imageid2, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img2_srclist[1] has larger RA value.
lightcurves_sorted_by_ra =[]
lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[0]])
lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[1]])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), 2)
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.per_timestep_variability_indices(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_many2manyflux_reduced_to_two_1to1(self):
"""
(See also assoc. test test_many2many_reduced_to_two_1to1 )
In this test-case we cross-associate between a rhombus of sources spread
about a central position, east-west in the first image,
north-south in the second.
The latter, north-south pair are slightly offset towards positive RA
and negative RA respectively.
The result is that the candidate associations are pruned down to
two one-to-one pairings..
"""
dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: n-m, ' + self._testMethodName})
n_images = 2
im_params = db_subs.example_dbimage_datasets(n_images)
centre_ra, centre_dec = 123., 10.5,
offset_deg = 20 / 3600. #20 arcsec
tiny_offset_deg = 1 / 3600. #1 arcsec
eastern_src = db_subs.example_extractedsource_tuple(
ra=centre_ra + offset_deg,
dec=centre_dec,
peak = 1.5, peak_err = 1e-1,
flux = 3.0, flux_err = 1e-1,)
western_src = db_subs.example_extractedsource_tuple(
ra=centre_ra - offset_deg,
dec=centre_dec,
peak = 1.7, peak_err = 1e-1,
flux = 3.2, flux_err = 1e-1,)
northern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra + tiny_offset_deg,
dec=centre_dec + offset_deg,
peak = 1.8, peak_err = 1e-1,
flux = 3.3, flux_err = 1e-1,
)
southern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra - tiny_offset_deg,
dec=centre_dec - offset_deg,
peak = 1.4, peak_err = 1e-1,
flux = 2.9, flux_err = 1e-1,)
# image 1
image1 = tkp.db.Image(database=self.database, dataset=dataset,
data=im_params[0])
dbgen.insert_extracted_sources(
image1.id, [eastern_src,western_src], 'blind')
associate_extracted_sources(image1.id, deRuiter_r = 3.717)
# image 2
image2 = tkp.db.Image(database=self.database, dataset=dataset,
data=im_params[1])
dbgen.insert_extracted_sources(
image2.id, [northern_source, southern_source], 'blind')
associate_extracted_sources(image2.id, deRuiter_r = 3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img1_srclist[1] has larger RA value.
lightcurves_sorted_by_ra =[]
lightcurves_sorted_by_ra.append( [western_src, southern_source])
lightcurves_sorted_by_ra.append( [eastern_src, northern_source])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra, r.wm_decl
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':dataset.id},
order='wm_ra,wm_decl')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.per_timestep_variability_indices(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_lightcurve(self):
# make 4 images with different date
images = []
for day in [3, 4, 5, 6]:
data = {'taustart_ts': datetime.datetime(2010, 3, day),
'tau_time': 3600,
'url': '/',
'freq_eff': 80e6,
'freq_bw': 1e6,
'beam_smaj_pix': float(2.7),
'beam_smin_pix': float(2.3),
'beam_pa_rad': float(1.7),
'deltax': float(-0.01111),
'deltay': float(0.01111),
'centre_ra': 111,
'centre_decl': 11,
'xtr_radius' : 3
}
image = Image(dataset=self.dataset, data=data)
images.append(image)
# 3 sources per image, with different coordinates & flux
data_list = []
for i in range(1, 4):
data_list.append({
'ra': 111.11 + i,
'decl': 11.11 + i,
'i_peak': 10. * i ,
'i_peak_err': 0.1,
})
# Insert the 3 sources in each image, while further varying the flux
lightcurves_sorted_by_ra = [[],[],[]]
for im_idx, image in enumerate(images):
# Create the "source finding results"
# Note that we reuse 'i_peak' as both peak & integrated flux.
img_sources = []
for src_idx, data in enumerate(data_list):
src = db_subs.example_extractedsource_tuple(
ra = data['ra'],dec=data['decl'],
peak=data['i_peak']* (1 + im_idx),
flux = data['i_peak']* (1 + im_idx)
)
lightcurves_sorted_by_ra[src_idx].append(src)
img_sources.append(src)
image.insert_extracted_sources(img_sources)
image.associate_extracted_sources(deRuiter_r=3.7)
# updates the dataset and its set of images
self.dataset.update()
self.dataset.update_images()
# update the images and their sets of sources
for image in self.dataset.images:
image.update()
image.update_sources()
# Now pick last image, select the first source (smallest RA)
# and extract its light curve
sources = self.dataset.images[-1].sources
sources = sorted(sources, key=attrgetter('ra'))
lightcurve = sources[0].lightcurve()
# check if the sources are associated in all images
self.assertEqual(len(images), len(lightcurve))
self.assertEqual(lightcurve[0][0], datetime.datetime(2010, 3, 3, 0, 0))
self.assertEqual(lightcurve[1][0], datetime.datetime(2010, 3, 4, 0, 0))
self.assertEqual(lightcurve[2][0], datetime.datetime(2010, 3, 5, 0, 0))
self.assertEqual(lightcurve[3][0], datetime.datetime(2010, 3, 6, 0, 0))
self.assertAlmostEqual(lightcurve[0][2], 10.)
self.assertAlmostEqual(lightcurve[1][2], 20.)
self.assertAlmostEqual(lightcurve[2][2], 30.)
self.assertAlmostEqual(lightcurve[3][2], 40.)
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': self.dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.per_timestep_variability_indices(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key],
places=5)