def test_basic_same_field_case(self):
""" Here we start with 1 source in image0.
We then add image1 (same field as image0), with a double association
for the source, and check assocskyrgn updates correctly.
"""
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
idx = 0
src_a = db_subs.example_extractedsource_tuple(
ra=im_params[idx]['centre_ra'],
dec=im_params[idx]['centre_decl'])
src_b = src_a._replace(ra=src_a.ra + 1. / 60.) # 1 arcminute offset
imgs = []
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
insert_extracted_sources(imgs[idx]._id, [src_a])
associate_extracted_sources(imgs[idx]._id, deRuiter_r, new_source_sigma_margin)
idx = 1
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
insert_extracted_sources(imgs[idx]._id, [src_a, src_b])
associate_extracted_sources(imgs[idx]._id, deRuiter_r, new_source_sigma_margin)
imgs[idx].update()
runcats = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id})
self.assertEqual(len(runcats), 2) #Just a sanity check.
skyassocs = columns_from_table('assocskyrgn',
where={'skyrgn':imgs[idx]._data['skyrgn']})
self.assertEqual(len(skyassocs), 2)
def insert_extracted_sources(self, results, extract="blind"):
"""Insert a list of sources
Args:
results (list): list of
utility.containers.ExtractionResult objects (as
returned from
sourcefinder.image.ImageData().extract()), or a list
of data tuples with the source information as follows:
(ra, dec,
ra_fit_err, dec_fit_err,
peak, peak_err,
flux, flux_err,
significance level,
beam major width (as), beam minor width(as),
beam parallactic angle
ew_sys_err, ns_sys_err,
error_radius).
extract (str):'blind', 'ff_nd' or 'ff_ms'
(see db.general.insert_extracted_sources)
"""
# To do: Figure out a saner method of passing the results around
# (Namedtuple, for starters?)
insert_extracted_sources(self._id, results=results, extract_type=extract)
def test_probably_not_a_transient(self):
"""
No source at 250MHz, but we detect a source at 50MHz.
Not necessarily a transient.
Should trivially ignore 250MHz data when looking at a new 50MHz source.
"""
img_params = self.img_params
img0 = img_params[0]
# This time around, we just manually exclude the steady src from
# the first image detections.
steady_low_freq_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve=defaultdict(lambda: self.always_detectable_flux))
# Insert first image, no sources.
tkp.db.Image(data=img_params[0], dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1], dataset=self.dataset)
xtr = steady_low_freq_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r,
self.new_source_sigma_margin)
transients = get_newsources_for_dataset(self.dataset.id)
# Should have no marked transients
self.assertEqual(len(transients), 0)
def test_probably_not_a_transient(self):
"""
No source at 250MHz, but we detect a source at 50MHz.
Not necessarily a transient.
Should trivially ignore 250MHz data when looking at a new 50MHz source.
"""
img_params = self.img_params
img0 = img_params[0]
# This time around, we just manually exclude the steady src from
# the first image detections.
steady_low_freq_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']
),
lightcurve=defaultdict(lambda :self.always_detectable_flux)
)
# Insert first image, no sources.
tkp.db.Image(data=img_params[0],dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1],dataset=self.dataset)
xtr = steady_low_freq_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r, self.new_source_sigma_margin)
transients = get_newsources_for_dataset(self.dataset.id)
# Should have no marked transients
self.assertEqual(len(transients), 0)
def insert_extracted_sources(self, results, extract='blind'):
"""Insert a list of sources
Args:
results (list): list of
utility.containers.ExtractionResult objects (as
returned from
sourcefinder.image.ImageData().extract()), or a list
of data tuples with the source information as follows:
(ra, dec,
ra_fit_err, dec_fit_err,
peak, peak_err,
flux, flux_err,
significance level,
beam major width (as), beam minor width(as),
beam parallactic angle
ew_sys_err, ns_sys_err,
error_radius).
extract (str):'blind', 'ff_nd' or 'ff_ms'
(see db.general.insert_extracted_sources)
"""
#To do: Figure out a saner method of passing the results around
# (Namedtuple, for starters?)
insert_extracted_sources(self._id, results=results, extract_type=extract)
def insert_and_associate_forced_fits(image_id,successful_fits,successful_ids):
assert len(successful_ids) == len(successful_fits)
nd_extractions=[]
nd_runcats=[]
ms_extractions=[]
ms_ids = []
for idx, id in enumerate(successful_ids):
if id[0] == 'ff_nd':
nd_extractions.append(successful_fits[idx])
nd_runcats.append(id[1])
elif id[0] == 'ff_ms':
ms_extractions.append(successful_fits[idx])
ms_ids.append(id[1])
else:
raise ValueError("Forced fit type id not recognised:" + id[0])
if nd_extractions:
logger.debug("adding null detections")
dbgen.insert_extracted_sources(image_id, nd_extractions,
extract_type='ff_nd',
ff_runcat_ids=nd_runcats)
dbnd.associate_nd(image_id)
else:
logger.debug("No successful nulldetection fits")
if ms_extractions:
dbgen.insert_extracted_sources(image_id, ms_extractions,
extract_type='ff_ms',
ff_monitor_ids=ms_ids)
logger.debug("adding monitoring sources")
dbmon.associate_ms(image_id)
else:
logger.debug("No successful monitor fits")
def test_infinite(self):
# Check that database insertion doesn't choke on infinite errors.
dataset = DataSet(data={'description': 'example dataset'},
database=self.database)
image = Image(dataset=dataset, data=db_subs.example_dbimage_data_dict())
# Inserting a standard example extractedsource should be fine
extracted_source = db_subs.example_extractedsource_tuple()
insert_extracted_sources(image._id, [extracted_source])
inserted = columns_from_table('extractedsource',
where= {'image' : image.id})
self.assertEqual(len(inserted), 1)
# But if the source has infinite errors we drop it and log a warning
extracted_source = db_subs.example_extractedsource_tuple(error_radius=float('inf'),
peak_err=float('inf'),
flux_err=float('inf'))
# We will add a handler to the root logger which catches all log
# output in a buffer.
iostream = BytesIO()
hdlr = logging.StreamHandler(iostream)
logging.getLogger().addHandler(hdlr)
insert_extracted_sources(image._id, [extracted_source])
logging.getLogger().removeHandler(hdlr)
# We want to be sure that the error has been appropriately logged.
self.assertIn("Dropped source fit with infinite flux errors",
iostream.getvalue())
inserted = columns_from_table('extractedsource',
where= {'image' : image.id})
self.assertEqual(len(inserted), 1)
def test_basic_same_field_case(self):
""" Here we start with 1 source in image0.
We then add image1 (same field as image0), with a double association
for the source, and check assocskyrgn updates correctly.
"""
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
idx = 0
src_a = db_subs.example_extractedsource_tuple(
ra=im_params[idx]['centre_ra'], dec=im_params[idx]['centre_decl'])
src_b = src_a._replace(ra=src_a.ra + 1. / 60.) # 1 arcminute offset
imgs = []
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
insert_extracted_sources(imgs[idx]._id, [src_a])
associate_extracted_sources(imgs[idx]._id, deRuiter_r,
new_source_sigma_margin)
idx = 1
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
insert_extracted_sources(imgs[idx]._id, [src_a, src_b])
associate_extracted_sources(imgs[idx]._id, deRuiter_r,
new_source_sigma_margin)
imgs[idx].update()
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 2) #Just a sanity check.
skyassocs = columns_from_table(
'assocskyrgn', where={'skyrgn': imgs[idx]._data['skyrgn']})
self.assertEqual(len(skyassocs), 2)
def test_two_field_overlap_new_transient(self):
"""Now for something more interesting - two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at 2nd timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
insert_extracted_sources(imgs[0]._id, [lower_steady_src, overlap_steady_src])
associate_extracted_sources(imgs[0]._id, deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
insert_extracted_sources(imgs[1]._id, [upper_steady_src, overlap_steady_src,
overlap_transient])
associate_extracted_sources(imgs[1]._id, deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 0)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id,))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 1)
def test_marginal_transient(self):
"""
( flux1 > (rms_min0*(det0 + margin) )
but ( flux1 < (rms_max0*(det0 + margin) )
--> Possible transient
If it was in a region of rms_min, we would (almost certainly) have seen
it in the first image. So new source --> Possible transient.
But if it was in a region of rms_max, then perhaps we would have missed
it. In which case, new source --> Just seeing deeper.
Note that if we are tiling overlapping images, then the first time
a field is processed with image-centre at the edge of the old field,
we may get a bunch of unhelpful 'possible transients'.
Furthermore, this will pick up fluctuating sources near the
image-margins even with a fixed field of view.
But without a more complex store of image-rms-per-position, we cannot
do better.
Hopefully we can use a 'distance from centre' feature to separate out
the good and bad candidates in this case.
"""
img_params = self.img_params
#Must pick flux value carefully to fire correct logic branch:
marginal_transient_flux = self.reliably_detected_at_image_centre_flux
marginal_transient = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve={img_params[1]['taustart_ts'] : marginal_transient_flux}
)
# First, check that we've set up the test correctly
rms_min0 = img_params[0]['rms_min']
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_transient_flux <
rms_max0 * (det0 + self.new_source_sigma_margin))
self.assertTrue(marginal_transient_flux >
rms_min0 * (det0 + self.new_source_sigma_margin))
for pars in self.img_params:
img = tkp.db.Image(data=pars, dataset=self.dataset)
xtr = marginal_transient.simulate_extraction(img,
extraction_type='blind')
if xtr is not None:
insert_extracted_sources(img._id, [xtr], 'blind')
associate_extracted_sources(img._id, deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
# Should have one 'possible' transient
self.assertEqual(len(newsources), 1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] < self.new_source_sigma_margin)
def test_marginal_transient(self):
"""
( flux1 > (rms_min0*(det0 + margin) )
but ( flux1 < (rms_max0*(det0 + margin) )
--> Possible transient
If it was in a region of rms_min, we would (almost certainly) have seen
it in the first image. So new source --> Possible transient.
But if it was in a region of rms_max, then perhaps we would have missed
it. In which case, new source --> Just seeing deeper.
Note that if we are tiling overlapping images, then the first time
a field is processed with image-centre at the edge of the old field,
we may get a bunch of unhelpful 'possible transients'.
Furthermore, this will pick up fluctuating sources near the
image-margins even with a fixed field of view.
But without a more complex store of image-rms-per-position, we cannot
do better.
Hopefully we can use a 'distance from centre' feature to separate out
the good and bad candidates in this case.
"""
img_params = self.img_params
#Must pick flux value carefully to fire correct logic branch:
marginal_transient_flux = self.reliably_detected_at_image_centre_flux
marginal_transient = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve={img_params[1]['taustart_ts']: marginal_transient_flux})
# First, check that we've set up the test correctly
rms_min0 = img_params[0]['rms_min']
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_transient_flux < rms_max0 *
(det0 + self.new_source_sigma_margin))
self.assertTrue(marginal_transient_flux > rms_min0 *
(det0 + self.new_source_sigma_margin))
for pars in self.img_params:
img = tkp.db.Image(data=pars, dataset=self.dataset)
xtr = marginal_transient.simulate_extraction(
img, extraction_type='blind')
if xtr is not None:
insert_extracted_sources(img._id, [xtr], 'blind')
associate_extracted_sources(img._id, deRuiter_r,
self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
# Should have one 'possible' transient
self.assertEqual(len(newsources), 1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] < self.new_source_sigma_margin)
def test_single_fixed_source(self):
"""test_single_fixed_source
- Pretend to extract the same source in each of a series of images.
- Perform source association
- Check the image source listing works
- Check runcat, assocxtrsource.
"""
fixed_src_runcat_id = None
for img_idx, im in enumerate(self.im_params):
self.db_imgs.append( Image(data=im, dataset=self.dataset))
last_img = self.db_imgs[-1]
insert_extracted_sources(last_img._id,
[db_subs.example_extractedsource_tuple()],'blind')
associate_extracted_sources(last_img._id, deRuiter_r,
new_source_sigma_margin)
running_cat = columns_from_table(table="runningcatalog",
keywords=['id', 'datapoints'],
where={"dataset":self.dataset.id})
self.assertEqual(len(running_cat), 1)
self.assertEqual(running_cat[0]['datapoints'], img_idx+1)
# Check runcat ID does not change for a steady single source
if img_idx == 0:
fixed_src_runcat_id = running_cat[0]['id']
self.assertIsNotNone(fixed_src_runcat_id, "No runcat id assigned to source")
self.assertEqual(running_cat[0]['id'], fixed_src_runcat_id,
"Multiple runcat ids for same fixed source")
runcat_flux = columns_from_table(table="runningcatalog_flux",
keywords=['f_datapoints'],
where={"runcat":fixed_src_runcat_id})
self.assertEqual(len(runcat_flux),1)
self.assertEqual(img_idx+1, runcat_flux[0]['f_datapoints'])
last_img.update()
last_img.update_sources()
img_xtrsrc_ids = [src.id for src in last_img.sources]
self.assertEqual(len(img_xtrsrc_ids), 1)
#Get the association row for most recent extraction:
assocxtrsrcs_rows = columns_from_table(table="assocxtrsource",
keywords=['runcat', 'xtrsrc' ],
where={"xtrsrc":img_xtrsrc_ids[0]})
# print "ImageID:", last_img.id
# print "Imgs sources:", img_xtrsrc_ids
# print "Assoc entries:", assocxtrsrcs_rows
# print "First extracted source id:", ds_source_ids[0]
# if len(assocxtrsrcs_rows):
# print "Associated source:", assocxtrsrcs_rows[0]['xtrsrc']
self.assertEqual(len(assocxtrsrcs_rows),1,
msg="No entries in assocxtrsrcs for image number "+str(img_idx))
self.assertEqual(assocxtrsrcs_rows[0]['runcat'], fixed_src_runcat_id,
"Mismatched runcat id in assocxtrsrc table")
def test_two_field_overlap_nulling_src(self):
"""Similar to above, but one source disappears:
Two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at *1st* timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
insert_extracted_sources(
imgs[0]._id,
[lower_steady_src, overlap_steady_src, overlap_transient])
associate_extracted_sources(
imgs[0]._id,
deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
insert_extracted_sources(imgs[1]._id,
[upper_steady_src, overlap_steady_src])
associate_extracted_sources(
imgs[1]._id,
deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
#This time we don't expect to get an immediate transient detection,
#but we *do* expect to get a null-source forced extraction request:
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 1)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
def test_only_first_epoch_source(self):
"""test_only_first_epoch_source
- Pretend to extract a source only from the first image.
- Run source association for each image, as we would in TraP.
- Check the image source listing works
- Check runcat and assocxtrsource are correct.
"""
first_epoch = True
extracted_source_ids = []
for im in self.im_params:
self.db_imgs.append(Image( data=im, dataset=self.dataset))
last_img = self.db_imgs[-1]
if first_epoch:
insert_extracted_sources(last_img._id,
[db_subs.example_extractedsource_tuple()], 'blind')
associate_extracted_sources(last_img._id, deRuiter_r,
new_source_sigma_margin)
# First, check the runcat has been updated correctly
running_cat = columns_from_table(table="runningcatalog",
keywords=['datapoints'],
where={"dataset": self.dataset.id})
self.assertEqual(len(running_cat), 1)
self.assertEqual(running_cat[0]['datapoints'], 1)
last_img.update()
last_img.update_sources()
img_xtrsrc_ids = [src.id for src in last_img.sources]
if first_epoch:
self.assertEqual(len(img_xtrsrc_ids),1)
extracted_source_ids.extend(img_xtrsrc_ids)
assocxtrsrcs_rows = columns_from_table(table="assocxtrsource",
keywords=['runcat', 'xtrsrc' ],
where={"xtrsrc":img_xtrsrc_ids[0]})
self.assertEqual(len(assocxtrsrcs_rows),1)
self.assertEqual(assocxtrsrcs_rows[0]['xtrsrc'], img_xtrsrc_ids[0])
else:
self.assertEqual(len(img_xtrsrc_ids),0)
first_epoch=False
#Assocxtrsources still ok after multiple images?
self.assertEqual(len(extracted_source_ids),1)
assocxtrsrcs_rows = columns_from_table(table="assocxtrsource",
keywords=['runcat', 'xtrsrc' ],
where={"xtrsrc":extracted_source_ids[0]})
self.assertEqual(len(assocxtrsrcs_rows),1)
self.assertEqual(assocxtrsrcs_rows[0]['xtrsrc'], extracted_source_ids[0],
"Runcat xtrsrc entry must match the only extracted source")
def insert_image_and_simulated_sources(dataset,
image_params,
mock_sources,
new_source_sigma_margin,
deruiter_radius=3.7):
"""
Simulates the standard database image-and-source insertion logic using mock
sources.
Args:
dataset: The dataset object
image_params (dict): Contains the image properties.
mock_sources (list of MockSource): The mock sources to simulate.
new_source_sigma_margin (float): Parameter passed to source-association
routines.
deruiter_radius (float): Parameter passed to source-association
routines.
Returns:
3-tuple (image, list of blind extractions, list of forced fits).
"""
image = tkp.db.Image(data=image_params, dataset=dataset)
blind_extractions = []
for src in mock_sources:
xtr = src.simulate_extraction(image, extraction_type='blind')
if xtr is not None:
blind_extractions.append(xtr)
insert_extracted_sources(image._id, blind_extractions, 'blind')
associate_extracted_sources(
image._id,
deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_source_sigma_margin)
nd_ids_posns = nulldetections.get_nulldetections(image.id)
nd_posns = [(ra, decl) for ids, ra, decl in nd_ids_posns]
forced_fits = []
for posn in nd_posns:
for src in mock_sources:
eps = 1e-13
if (math.fabs(posn[0] - src.base_source.ra) < eps
and math.fabs(posn[1] - src.base_source.dec) < eps):
forced_fits.append(
src.simulate_extraction(image, extraction_type='ff_nd'))
if len(nd_posns) != len(forced_fits):
raise LookupError("Something went wrong, nulldetection position did "
"not match a mock source.")
#image.insert_extracted_sources(forced_fits, 'ff_nd')
dbgen.insert_extracted_sources(
image.id,
forced_fits,
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_posns])
nulldetections.associate_nd(image.id)
return image, blind_extractions, forced_fits
def store_extractions(images, extraction_results, job_config):
logger.debug("storing extracted sources to database")
# we also set the image max,min RMS values which calculated during
# source extraction
detection_thresh = job_config.source_extraction['detection_threshold']
analysis_thresh = job_config.source_extraction['analysis_threshold']
for (db_image, accessor), results in zip(images, extraction_results):
db_image.update(rms_min=results.rms_min, rms_max=results.rms_max,
detection_thresh=detection_thresh,
analysis_thresh=analysis_thresh)
dbgen.insert_extracted_sources(db_image.id, results.sources, 'blind')
def store_extractions(images, extraction_results, job_config):
logger.debug("storing extracted sources to database")
# we also set the image max,min RMS values which calculated during
# source extraction
detection_thresh = job_config.source_extraction['detection_threshold']
analysis_thresh = job_config.source_extraction['analysis_threshold']
for (db_image, accessor), results in zip(images, extraction_results):
db_image.update(rms_min=results.rms_min,
rms_max=results.rms_max,
detection_thresh=detection_thresh,
analysis_thresh=analysis_thresh)
dbgen.insert_extracted_sources(db_image.id, results.sources, 'blind')
def test_rejected_initial_image(self):
"""
An image which is rejected should not be taken into account when
deciding whether a patch of sky has been previously observed, and
hence whether any detections in that area are (potential) transients.
Here, we create a database with two images. The first
(choronologically) is rejected; the second contains a source. That
source should not be marked as a transient.
"""
dataset = tkp.db.DataSet(data={"description": "Trans:" + self._testMethodName}, database=tkp.db.Database())
# We use a dataset with two images
# NB the routine in db_subs automatically increments time between
# images.
n_images = 2
db_imgs = [
tkp.db.Image(data=im_params, dataset=dataset)
for im_params in db_subs.generate_timespaced_dbimages_data(n_images)
]
# The first image is rejected for an arbitrary reason
# (for the sake of argument, we use an unacceptable RMS).
db_quality.reject(
imageid=db_imgs[0].id,
reason=db_quality.reject_reasons["rms"],
comment=self._testMethodName,
session=self.session,
)
# Have to commit here: old DB code makes queries in a separate transaction.
self.session.commit()
# Since we rejected the first image, we only find a source in the
# second.
source = db_subs.example_extractedsource_tuple()
insert_extracted_sources(db_imgs[1]._id, [source])
# Standard source association procedure etc.
associate_extracted_sources(db_imgs[1].id, deRuiter_r=3.7, new_source_sigma_margin=3)
# Our source should _not_ be a transient. That is, there should be no
# entries in the newsource table for this dataset.
cursor = tkp.db.execute(
"""\
SELECT t.id FROM newsource t, runningcatalog rc
WHERE t.runcat = rc.id
AND rc.dataset = %(ds_id)s
""",
{"ds_id": dataset.id},
)
self.assertEqual(cursor.rowcount, 0)
def test_rejected_initial_image(self):
"""
An image which is rejected should not be taken into account when
deciding whether a patch of sky has been previously observed, and
hence whether any detections in that area are (potential) transients.
Here, we create a database with two images. The first
(choronologically) is rejected; the second contains a source. That
source should not be marked as a transient.
"""
dataset = tkp.db.DataSet(
data={'description': "Trans:" + self._testMethodName},
database=tkp.db.Database())
# We use a dataset with two images
# NB the routine in db_subs automatically increments time between
# images.
n_images = 2
db_imgs = [
tkp.db.Image(data=im_params, dataset=dataset) for im_params in
db_subs.generate_timespaced_dbimages_data(n_images)
]
# The first image is rejected for an arbitrary reason
# (for the sake of argument, we use an unacceptable RMS).
db_quality.reject(imageid=db_imgs[0].id,
reason=db_quality.reject_reasons['rms'],
comment=self._testMethodName,
session=self.session)
# Have to commit here: old DB code makes queries in a separate transaction.
self.session.commit()
# Since we rejected the first image, we only find a source in the
# second.
source = db_subs.example_extractedsource_tuple()
insert_extracted_sources(db_imgs[1]._id, [source])
# Standard source association procedure etc.
associate_extracted_sources(db_imgs[1].id,
deRuiter_r=3.7,
new_source_sigma_margin=3)
# Our source should _not_ be a transient. That is, there should be no
# entries in the newsource table for this dataset.
cursor = tkp.db.execute(
"""\
SELECT t.id FROM newsource t, runningcatalog rc
WHERE t.runcat = rc.id
AND rc.dataset = %(ds_id)s
""", {"ds_id": dataset.id})
self.assertEqual(cursor.rowcount, 0)
def test_new_skyregion_insertion(self):
"""Here we test the association logic executed upon insertion of a
new skyregion.
We expect that any pre-existing entries in the runningcatalog
which lie within the field of view will be marked as
'within this region', through the presence of an entry in table
``assocskyrgn``.
Conversely sources outside the FoV should not be marked as related.
We begin with img0, with a source at centre.
Then we add 2 more (empty) images/fields at varying positions.
"""
n_images = 6
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
src_in_img0 = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
# First image
image0 = tkp.db.Image(dataset=self.dataset, data=im_params[0])
insert_extracted_sources(image0._id, [src_in_img0])
associate_extracted_sources(image0._id, deRuiter_r,
new_source_sigma_margin)
image0.update()
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 1) #Just a sanity check.
##Second, different *But overlapping* image:
idx = 1
im_params[idx]['centre_decl'] += im_params[idx]['xtr_radius'] * 0.9
image1 = tkp.db.Image(dataset=self.dataset, data=im_params[idx])
image1.update()
assocs = columns_from_table('assocskyrgn',
where={'skyrgn': image1._data['skyrgn']})
self.assertEqual(len(assocs), 1)
self.assertEqual(assocs[0]['runcat'], runcats[0]['id'])
##Third, different *and NOT overlapping* image:
idx = 2
im_params[idx]['centre_decl'] += im_params[idx]['xtr_radius'] * 1.1
image2 = tkp.db.Image(dataset=self.dataset, data=im_params[idx])
image2.update()
assocs = columns_from_table('assocskyrgn',
where={'skyrgn': image2._data['skyrgn']})
self.assertEqual(len(assocs), 0)
def test_two_field_overlap_nulling_src(self):
"""Similar to above, but one source disappears:
Two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at *1st* timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
insert_extracted_sources(imgs[0]._id, [lower_steady_src, overlap_steady_src,
overlap_transient])
associate_extracted_sources(imgs[0]._id, deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
insert_extracted_sources(imgs[1]._id, [upper_steady_src, overlap_steady_src])
associate_extracted_sources(imgs[1]._id, deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
#This time we don't expect to get an immediate transient detection,
#but we *do* expect to get a null-source forced extraction request:
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 1)
runcats = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
def insert_image_and_simulated_sources(dataset, image_params, mock_sources,
new_source_sigma_margin,
deruiter_radius=3.7):
"""
Simulates the standard database image-and-source insertion logic using mock
sources.
Args:
dataset: The dataset object
image_params (dict): Contains the image properties.
mock_sources (list of MockSource): The mock sources to simulate.
new_source_sigma_margin (float): Parameter passed to source-association
routines.
deruiter_radius (float): Parameter passed to source-association
routines.
Returns:
3-tuple (image, list of blind extractions, list of forced fits).
"""
image = tkp.db.Image(data=image_params,dataset=dataset)
blind_extractions=[]
for src in mock_sources:
xtr = src.simulate_extraction(image,extraction_type='blind')
if xtr is not None:
blind_extractions.append(xtr)
image.insert_extracted_sources(blind_extractions,'blind')
image.associate_extracted_sources(deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_source_sigma_margin)
nd_ids_posns = nulldetections.get_nulldetections(image.id)
nd_posns = [(ra,decl) for ids, ra, decl in nd_ids_posns]
forced_fits = []
for posn in nd_posns:
for src in mock_sources:
eps = 1e-13
if (math.fabs(posn[0] - src.base_source.ra)<eps and
math.fabs(posn[1] - src.base_source.dec)<eps ):
forced_fits.append(
src.simulate_extraction(image,extraction_type='ff_nd')
)
if len(nd_posns) != len(forced_fits):
raise LookupError("Something went wrong, nulldetection position did "
"not match a mock source.")
#image.insert_extracted_sources(forced_fits, 'ff_nd')
dbgen.insert_extracted_sources(image.id, forced_fits, 'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_posns])
nulldetections.associate_nd(image.id)
return image, blind_extractions, forced_fits
def test_new_skyregion_insertion(self):
"""Here we test the association logic executed upon insertion of a
new skyregion.
We expect that any pre-existing entries in the runningcatalog
which lie within the field of view will be marked as
'within this region', through the presence of an entry in table
``assocskyrgn``.
Conversely sources outside the FoV should not be marked as related.
We begin with img0, with a source at centre.
Then we add 2 more (empty) images/fields at varying positions.
"""
n_images = 6
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
src_in_img0 = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],)
# First image
image0 = tkp.db.Image(dataset=self.dataset, data=im_params[0])
insert_extracted_sources(image0._id, [src_in_img0])
associate_extracted_sources(image0._id, deRuiter_r, new_source_sigma_margin)
image0.update()
runcats = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id})
self.assertEqual(len(runcats), 1) #Just a sanity check.
##Second, different *But overlapping* image:
idx = 1
im_params[idx]['centre_decl'] += im_params[idx]['xtr_radius'] * 0.9
image1 = tkp.db.Image(dataset=self.dataset, data=im_params[idx])
image1.update()
assocs = columns_from_table('assocskyrgn',
where={'skyrgn':image1._data['skyrgn']})
self.assertEqual(len(assocs), 1)
self.assertEqual(assocs[0]['runcat'], runcats[0]['id'])
##Third, different *and NOT overlapping* image:
idx = 2
im_params[idx]['centre_decl'] += im_params[idx]['xtr_radius'] * 1.1
image2 = tkp.db.Image(dataset=self.dataset, data=im_params[idx])
image2.update()
assocs = columns_from_table('assocskyrgn',
where={'skyrgn':image2._data['skyrgn']})
self.assertEqual(len(assocs), 0)
def test_null_case_sequential(self):
"""test_null_case_sequential
-Check extractedsource insertion routines can deal with empty input!
-Check source association can too
"""
for im in self.im_params:
self.db_imgs.append(Image(data=im, dataset=self.dataset))
insert_extracted_sources(self.db_imgs[-1]._id, [],'blind')
associate_extracted_sources(self.db_imgs[-1]._id, deRuiter_r,
new_source_sigma_margin)
running_cat = columns_from_table(table="runningcatalog",
keywords="*",
where={"dataset":self.dataset.id})
self.assertEqual(len(running_cat), 0)
def test_null_case_sequential(self):
"""test_null_case_sequential
-Check extractedsource insertion routines can deal with empty input!
-Check source association can too
"""
for im in self.im_params:
self.db_imgs.append(Image(data=im, dataset=self.dataset))
insert_extracted_sources(self.db_imgs[-1]._id, [], 'blind')
associate_extracted_sources(self.db_imgs[-1]._id, deRuiter_r,
new_source_sigma_margin)
running_cat = columns_from_table(
table="runningcatalog",
keywords="*",
where={"dataset": self.dataset.id})
self.assertEqual(len(running_cat), 0)
def test_one2oneflux(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-1'})
n_images = 3
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
src_list = []
src = db_subs.example_extractedsource_tuple()
src0 = src._replace(flux=2.0)
src_list.append(src0)
src1 = src._replace(flux=2.5)
src_list.append(src1)
src2 = src._replace(flux=2.4)
src_list.append(src2)
for idx, im in enumerate(im_params):
image = tkp.db.Image(database=self.database,
dataset=dataset,
data=im)
insert_extracted_sources(image._id, [src_list[idx]])
associate_extracted_sources(image.id, deRuiter_r=3.717)
query = """\
SELECT rf.avg_f_int
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
result = zip(*self.database.cursor.fetchall())
avg_f_int = result[0]
self.assertEqual(len(avg_f_int), 1)
py_metrics = db_subs.lightcurve_metrics(src_list)
self.assertAlmostEqual(avg_f_int[0], py_metrics[-1]['avg_f_int'])
runcat_id = columns_from_table('runningcatalog',
where={'dataset': dataset.id})
self.assertEqual(len(runcat_id), 1)
runcat_id = runcat_id[0]['id']
# Check evolution of variability indices
db_metrics = db_queries.get_assoc_entries(self.database, runcat_id)
self.assertEqual(len(db_metrics), n_images)
# Compare the python- and db-calculated values
for i in range(len(db_metrics)):
for key in ('v_int', 'eta_int'):
self.assertAlmostEqual(db_metrics[i][key], py_metrics[i][key])
def test_probably_not_a_transient(self):
"""
( flux1 < (rms_min0*(det0 + margin) )
--> Probably not a transient
NB even if
avg_source_flux == rms_min0*det0 + epsilon
we might not detect it in the
first image, due to noise fluctuations. So we provide the
user-tunable marginal_detection_thresh, to ignore these 'noise'
transients.
"""
img_params = self.img_params
img0 = img_params[0]
marginal_steady_src_flux = self.barely_detectable_flux
# This time around, we just manually exclude the steady src from
# the first image detections.
marginal_steady_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']
),
lightcurve=defaultdict(lambda :marginal_steady_src_flux)
)
# First, check that we've set up the test correctly
rms_min0 = img_params[0]['rms_min']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_steady_src_flux <
rms_min0 * (det0 + self.new_source_sigma_margin))
# Insert first image, no sources.
tkp.db.Image(data=img_params[0], dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1], dataset=self.dataset)
xtr = marginal_steady_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
# Should have no flagged new sources
self.assertEqual(len(newsources), 0)
def test_one2oneflux(self):
dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: 1-1'})
n_images = 3
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
src_list = []
src = db_subs.example_extractedsource_tuple()
src0 = src._replace(flux=2.0)
src_list.append(src0)
src1 = src._replace(flux=2.5)
src_list.append(src1)
src2 = src._replace(flux=2.4)
src_list.append(src2)
for idx, im in enumerate(im_params):
image = tkp.db.Image(database=self.database, dataset=dataset, data=im)
insert_extracted_sources(image._id, [src_list[idx]])
associate_extracted_sources(image.id, deRuiter_r=3.717)
query = """\
SELECT rf.avg_f_int
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
result = zip(*self.database.cursor.fetchall())
avg_f_int = result[0]
self.assertEqual(len(avg_f_int), 1)
py_metrics = db_subs.lightcurve_metrics(src_list)
self.assertAlmostEqual(avg_f_int[0], py_metrics[-1]['avg_f_int'])
runcat_id = columns_from_table('runningcatalog',
where={'dataset':dataset.id})
self.assertEqual(len(runcat_id),1)
runcat_id = runcat_id[0]['id']
# Check evolution of variability indices
db_metrics = db_queries.get_assoc_entries(self.database,
runcat_id)
self.assertEqual(len(db_metrics), n_images)
# Compare the python- and db-calculated values
for i in range(len(db_metrics)):
for key in ('v_int','eta_int'):
self.assertAlmostEqual(db_metrics[i][key], py_metrics[i][key])
def test_two_field_basic_case(self):
"""
Here we create 2 disjoint image fields, with one source at centre of
each, and check that the second source inserted does not get flagged as
newsource.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 2 + 0.5
imgs = []
for idx in range(len(im_params)):
imgs.append(tkp.db.Image(dataset=self.dataset,
data=im_params[idx]))
for idx in range(len(im_params)):
central_src = db_subs.example_extractedsource_tuple(
ra=im_params[idx]['centre_ra'],
dec=im_params[idx]['centre_decl'])
imgs.append(tkp.db.Image(dataset=self.dataset,
data=im_params[idx]))
insert_extracted_sources(imgs[idx]._id, [central_src])
associate_extracted_sources(imgs[idx]._id, deRuiter_r,
new_source_sigma_margin)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 2) #Just a sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id, ))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 0)
def test_probably_not_a_transient(self):
"""
( flux1 < (rms_min0*(det0 + margin) )
--> Probably not a transient
NB even if
avg_source_flux == rms_min0*det0 + epsilon
we might not detect it in the
first image, due to noise fluctuations. So we provide the
user-tunable marginal_detection_thresh, to ignore these 'noise'
transients.
"""
img_params = self.img_params
img0 = img_params[0]
marginal_steady_src_flux = self.barely_detectable_flux
# This time around, we just manually exclude the steady src from
# the first image detections.
marginal_steady_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve=defaultdict(lambda: marginal_steady_src_flux))
# First, check that we've set up the test correctly
rms_min0 = img_params[0]['rms_min']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_steady_src_flux < rms_min0 *
(det0 + self.new_source_sigma_margin))
# Insert first image, no sources.
tkp.db.Image(data=img_params[0], dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1], dataset=self.dataset)
xtr = marginal_steady_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r,
self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
# Should have no flagged new sources
self.assertEqual(len(newsources), 0)
def test_one2one_flux_infinite_error(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-1'})
n_images = 3
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
src_list = []
src = db_subs.example_extractedsource_tuple()
src0 = src._replace(flux=2.0)
src_list.append(src0)
src1 = src._replace(flux=2.5)
src_list.append(src1)
src2 = src._replace(flux=0.0001,
flux_err=float('inf'),
peak=0.0001,
peak_err=float('inf'))
src_list.append(src2)
for idx, im in enumerate(im_params):
image = tkp.db.Image(database=self.database,
dataset=dataset,
data=im)
insert_extracted_sources(image._id, [src_list[idx]])
associate_extracted_sources(image.id, deRuiter_r=3.717)
query = """\
SELECT rf.avg_f_int
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
"""
cursor = tkp.db.execute(query, {'dataset': dataset.id})
results = db_subs.get_db_rows_as_dicts(cursor)
self.assertEqual(len(results), 1)
self.assertEqual(results[0]['f_datapoints'], 2)
self.assertAlmostEqual(results[0]['avg_f_int'],
(src0.flux + src1.flux) / 2.0)
def test_two_field_basic_case(self):
"""
Here we create 2 disjoint image fields, with one source at centre of
each, and check that the second source inserted does not get flagged as
newsource.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 2 + 0.5
imgs = []
for idx in range(len(im_params)):
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
for idx in range(len(im_params)):
central_src = db_subs.example_extractedsource_tuple(
ra=im_params[idx]['centre_ra'],
dec=im_params[idx]['centre_decl'])
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[idx]))
insert_extracted_sources(imgs[idx]._id, [central_src])
associate_extracted_sources(imgs[idx]._id, deRuiter_r, new_source_sigma_margin)
runcats = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id})
self.assertEqual(len(runcats), 2) #Just a sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id,))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 0)
def test_certain_transient(self):
"""
flux1 > (rms_max0*(det0+margin)
--> Definite transient
Nice and bright, must be new - mark it definite transient.
"""
img_params = self.img_params
bright_transient = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve={img_params[1]['taustart_ts']:
self.always_detectable_flux}
)
#First, check that we've set up the test correctly:
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(bright_transient.lightcurve.values()[0] >
rms_max0*(det0 + self.new_source_sigma_margin ) )
for pars in self.img_params:
img = tkp.db.Image(data=pars,dataset=self.dataset)
xtr = bright_transient.simulate_extraction(img,
extraction_type='blind')
if xtr is not None:
insert_extracted_sources(img._id, [xtr], 'blind')
associate_extracted_sources(img._id, deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
#Should have one 'definite' transient
self.assertEqual(len(newsources),1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['low_thresh_sigma'] >
newsources[0]['high_thresh_sigma'])
def test_certain_transient(self):
"""
flux1 > (rms_max0*(det0+margin)
--> Definite transient
Nice and bright, must be new - mark it definite transient.
"""
img_params = self.img_params
bright_transient = MockSource(example_extractedsource_tuple(
ra=img_params[0]['centre_ra'], dec=img_params[0]['centre_decl']),
lightcurve={
img_params[1]['taustart_ts']:
self.always_detectable_flux
})
#First, check that we've set up the test correctly:
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(bright_transient.lightcurve.values()[0] > rms_max0 *
(det0 + self.new_source_sigma_margin))
for pars in self.img_params:
img = tkp.db.Image(data=pars, dataset=self.dataset)
xtr = bright_transient.simulate_extraction(img,
extraction_type='blind')
if xtr is not None:
insert_extracted_sources(img._id, [xtr], 'blind')
associate_extracted_sources(img._id, deRuiter_r,
self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
#Should have one 'definite' transient
self.assertEqual(len(newsources), 1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(newsources[0]['low_thresh_sigma'] > newsources[0]
['high_thresh_sigma'])
def insert_and_associate_forced_fits(image_id, successful_fits,
successful_ids):
assert len(successful_ids) == len(successful_fits)
nd_extractions = []
nd_runcats = []
ms_extractions = []
ms_ids = []
for idx, id in enumerate(successful_ids):
if id[0] == 'ff_nd':
nd_extractions.append(successful_fits[idx])
nd_runcats.append(id[1])
elif id[0] == 'ff_ms':
ms_extractions.append(successful_fits[idx])
ms_ids.append(id[1])
else:
raise ValueError("Forced fit type id not recognised:" + id[0])
if nd_extractions:
logger.info("adding null detections")
dbgen.insert_extracted_sources(image_id,
nd_extractions,
extract_type='ff_nd',
ff_runcat_ids=nd_runcats)
dbnd.associate_nd(image_id)
else:
logger.info("No successful nulldetection fits")
if ms_extractions:
dbgen.insert_extracted_sources(image_id,
ms_extractions,
extract_type='ff_ms',
ff_monitor_ids=ms_ids)
logger.info("adding monitoring sources")
dbmon.associate_ms(image_id)
else:
logger.info("No successful monitor fits")
def test_infinite(self):
# Check that database insertion doesn't choke on infinite errors.
dataset = DataSet(data={'description': 'example dataset'},
database=self.database)
image = Image(dataset=dataset,
data=db_subs.example_dbimage_data_dict())
# Inserting a standard example extractedsource should be fine
extracted_source = db_subs.example_extractedsource_tuple()
insert_extracted_sources(image._id, [extracted_source])
inserted = columns_from_table('extractedsource',
where={'image': image.id})
self.assertEqual(len(inserted), 1)
# But if the source has infinite errors we drop it and log a warning
extracted_source = db_subs.example_extractedsource_tuple(
error_radius=float('inf'),
peak_err=float('inf'),
flux_err=float('inf'))
# We will add a handler to the root logger which catches all log
# output in a buffer.
iostream = BytesIO()
hdlr = logging.StreamHandler(iostream)
logging.getLogger().addHandler(hdlr)
insert_extracted_sources(image._id, [extracted_source])
logging.getLogger().removeHandler(hdlr)
# We want to be sure that the error has been appropriately logged.
self.assertIn("Dropped source fit with infinite flux errors",
iostream.getvalue())
inserted = columns_from_table('extractedsource',
where={'image': image.id})
self.assertEqual(len(inserted), 1)
def test_one2one_flux_infinite_error(self):
dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: 1-1'})
n_images = 3
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
src_list = []
src = db_subs.example_extractedsource_tuple()
src0 = src._replace(flux=2.0)
src_list.append(src0)
src1 = src._replace(flux=2.5)
src_list.append(src1)
src2 = src._replace(flux=0.0001, flux_err=float('inf'),
peak=0.0001, peak_err=float('inf'))
src_list.append(src2)
for idx, im in enumerate(im_params):
image = tkp.db.Image(database=self.database, dataset=dataset, data=im)
insert_extracted_sources(image._id, [src_list[idx]])
associate_extracted_sources(image.id, deRuiter_r=3.717)
query = """\
SELECT rf.avg_f_int
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
"""
cursor = tkp.db.execute(query, {'dataset': dataset.id})
results = db_subs.get_db_rows_as_dicts(cursor)
self.assertEqual(len(results),1)
self.assertEqual(results[0]['f_datapoints'],2)
self.assertAlmostEqual(results[0]['avg_f_int'],
(src0.flux + src1.flux)/2.0 )
def test_monitoringSource(self):
data = {'description': "monitoringlist:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, 1 band -> 3 images.
taustart_tss = [datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)]
#freq_effs = [124, 149, 156, 185]
freq_effs = [124]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.example_dbimage_datasets(len(freq_effs)
* len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image.
# We have one to be monitored source and one "normal" source
src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5)
src1_mon = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5)
# Group images in blocks of 1, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images),) * len(freq_effs)):
for image in images:
# The "normal" source is seen at all timesteps
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
for image in images:
dbass.associate_extracted_sources(image.id, deRuiter_r=5.68)
# The monitoring sources are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0_mon
# src1_mon is the monitoring source at all timesteps
dbgen.insert_extracted_sources(image.id, [src1_mon], 'ff_ms')
# And here we have to associate the monitoring sources with the
# runcat sources...
dbmon.associate_ms(image.id)
query = """\
SELECT id
,mon_src
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
AND datapoints = 3
ORDER BY id
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, one for the "normal"
# source and one for the monitoring source.
# Both should have three datapoints.
print "dp_result:",result
self.assertEqual(len(result), 2)
# The first source is the "normal" one
self.assertEqual(result[0][1], False)
# The second source is the monitoring one
self.assertEqual(result[1][1], True)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog_flux entries,
# one for every source, where every source has
# three f_datapoints
self.assertEqual(len(result), 2)
# "Normal" source: three flux datapoints
self.assertEqual(result[0][1], False)
self.assertEqual(result[0][2], 3)
# Monitoring source: three flux datapoints
self.assertEqual(result[1][1], True)
self.assertEqual(result[1][2], 3)
# We should also have two lightcurves for both sources,
# where both sources have three datapoints.
# The association types of the "normal" source are
# 3 (first) or 4 (later ones), while the monitoring source
# association types are 8 (first) or 9 (later ones).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.taustart_ts
,r.mon_src
,x.extract_type
FROM assocxtrsource a
,extractedsource x
,image i
,runningcatalog r
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
AND a.runcat = r.id
ORDER BY a.runcat
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 3 + 3 entries for source 1 and 2 resp.
self.assertEqual(len(result), 6)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][3], taustart_tss[0])
self.assertEqual(result[0][4], False)
self.assertEqual(result[0][5], 0)
# It was known at second timestep
self.assertEqual(result[1][2], 3)
self.assertEqual(result[1][3], taustart_tss[1])
self.assertEqual(result[1][4], result[0][4])
self.assertEqual(result[1][5], result[0][5])
# It was known at third timestep
self.assertEqual(result[2][2], result[1][2])
self.assertEqual(result[2][3], taustart_tss[2])
self.assertEqual(result[2][4], result[1][4])
self.assertEqual(result[2][5], result[1][5])
# The individual light-curve datapoints for the monitoring source
# It was new at first timestep
self.assertEqual(result[3][2], 8)
self.assertEqual(result[3][3], taustart_tss[0])
self.assertEqual(result[3][4], True)
self.assertEqual(result[3][5], 2)
# It was known at second timestep
self.assertEqual(result[4][2], 9)
self.assertEqual(result[4][3], taustart_tss[1])
self.assertEqual(result[4][4], result[3][4])
self.assertEqual(result[4][5], result[3][5])
# It was known at third timestep
self.assertEqual(result[5][2], result[4][2])
self.assertEqual(result[5][3], taustart_tss[2])
self.assertEqual(result[5][4], result[4][4])
self.assertEqual(result[5][5], result[4][5])
def run(job_name, supplied_mon_coords=[]):
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
# get parallelise props. Defaults to multiproc with autodetect num cores
parallelise = pipe_config.get('parallelise', {})
distributor = os.environ.get('TKP_PARALLELISE', parallelise.get('method',
'multiproc'))
runner = Runner(distributor=distributor,
cores=parallelise.get('cores', 0))
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
beamwidths_limit = job_config.association.beamwidths_limit
new_src_sigma = job_config.transient_search.new_source_sigma_margin
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
if supplied_mon_coords:
dbgen.insert_monitor_positions(dataset_id,supplied_mon_coords)
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
if supplied_mon_coords:
logger.warn("Monitor positions supplied will be ignored. "
"(Previous dataset specified)")
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
rms_est_sigma = job_config.persistence.rms_est_sigma
rms_est_fraction = job_config.persistence.rms_est_fraction
metadatas = runner.map("persistence_node_step", imgs,
[image_cache_params, rms_est_sigma, rms_est_fraction])
metadatas = [m[0] for m in metadatas if m]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner.map("quality_reject_check", urls, arguments)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extraction_results = runner.map("extract_sources", urls, arguments)
logger.info("storing extracted sources to database")
# we also set the image max,min RMS values which calculated during
# source extraction
for image, results in zip(images, extraction_results):
image.update(rms_min=results.rms_min, rms_max=results.rms_max,
detection_thresh=se_parset['detection_threshold'],
analysis_thresh=se_parset['analysis_threshold'])
dbgen.insert_extracted_sources(image.id, results.sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_src_sigma)
all_fit_posns, all_fit_ids = steps_ff.get_forced_fit_requests(image)
if all_fit_posns:
successful_fits, successful_ids = steps_ff.perform_forced_fits(
all_fit_posns, all_fit_ids, image.url, se_parset)
steps_ff.insert_and_associate_forced_fits(image.id,successful_fits,
successful_ids)
dbgen.update_dataset_process_end_ts(dataset_id)
def run(job_name, supplied_mon_coords=[]):
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
# get parallelise props. Defaults to multiproc with autodetect num cores
parallelise = pipe_config.get('parallelise', {})
distributor = os.environ.get('TKP_PARALLELISE', parallelise.get('method',
'multiproc'))
runner = Runner(distributor=distributor,
cores=parallelise.get('cores', 0))
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
beamwidths_limit = job_config.association.beamwidths_limit
new_src_sigma = job_config.transient_search.new_source_sigma_margin
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
if supplied_mon_coords:
dbgen.insert_monitor_positions(dataset_id,supplied_mon_coords)
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
if supplied_mon_coords:
logger.warn("Monitor positions supplied will be ignored. "
"(Previous dataset specified)")
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
rms_est_sigma = job_config.persistence.rms_est_sigma
rms_est_fraction = job_config.persistence.rms_est_fraction
metadatas = runner.map("persistence_node_step", imgs,
[image_cache_params, rms_est_sigma, rms_est_fraction])
metadatas = [m[0] for m in metadatas if m]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner.map("quality_reject_check", urls, arguments)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extraction_results = runner.map("extract_sources", urls, arguments)
logger.info("storing extracted sources to database")
# we also set the image max,min RMS values which calculated during
# source extraction
for image, results in zip(images, extraction_results):
image.update(rms_min=results.rms_min, rms_max=results.rms_max,
detection_thresh=se_parset['detection_threshold'],
analysis_thresh=se_parset['analysis_threshold'])
dbgen.insert_extracted_sources(image.id, results.sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_src_sigma)
expiration = job_config.source_extraction.expiration
all_fit_posns, all_fit_ids = steps_ff.get_forced_fit_requests(image,
expiration)
if all_fit_posns:
successful_fits, successful_ids = steps_ff.perform_forced_fits(
all_fit_posns, all_fit_ids, image.url, se_parset)
steps_ff.insert_and_associate_forced_fits(image.id,successful_fits,
successful_ids)
dbgen.update_dataset_process_end_ts(dataset_id)
logger.info("calculating variability metrics")
execute_store_varmetric(dataset_id)
def test_two_field_overlap_new_transient(self):
"""Now for something more interesting - two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at 2nd timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
insert_extracted_sources(imgs[0]._id,
[lower_steady_src, overlap_steady_src])
associate_extracted_sources(
imgs[0]._id,
deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
insert_extracted_sources(
imgs[1]._id,
[upper_steady_src, overlap_steady_src, overlap_transient])
associate_extracted_sources(
imgs[1]._id,
deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 0)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id, ))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 1)
def test_m2m_nullDetection(self):
"""
This tests that two sources (close-by to be associated if they were
detected at different timesteps) which are not seen in the next
image and thus have forced fits, will have separate light curves.
The postions are from the previous test.
"""
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, just 1 band -> 3 images.
taustart_tss = [
datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)
]
freq_effs = [124]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.generate_timespaced_dbimages_data(
len(freq_effs) * len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(
im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image
# and close together (see previous test)
src0 = db_subs.example_extractedsource_tuple(ra=122.985, dec=10.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.015, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images), ) * len(freq_effs)):
for image in images:
# The sources are only seen at timestep 0
if (image.taustart_ts == taustart_tss[0]):
dbgen.insert_extracted_sources(image.id, [src0, src1],
'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id,
deRuiter_r=5.68,
new_source_sigma_margin=3)
nd_ids_pos = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(nd_ids_pos), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 & src1 are null detections at the second timestep
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(
image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
else:
# All other images have two null detections.
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(
image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog_flux entries,
# one for every source in the band, i.e. 2 x 1.
self.assertEqual(len(result), 2)
# Source 0: inserted into timestep 0.
# Force-fits in images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0
# Force-fits in images at next timesteps.
# so 1+2 for bands 0
self.assertEqual(result[1][2], 3)
#self.assertEqual(result[2][2], 2)
#self.assertEqual(result[3][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3).
# Source 2 also has 3 datapoints for band0 (t1,t2,t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 3 + 3 entries for source 0 and 1 resp.
self.assertEqual(len(result), 6)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source2: new at t1, band0
self.assertEqual(result[3][2], 4)
self.assertEqual(result[3][3], result[1][3])
self.assertEqual(result[3][4], taustart_tss[0])
# Source2: Forced fit at t2, band0
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[1])
# Source2: Forced fit at t3, band0
self.assertEqual(result[5][2], 7)
self.assertEqual(result[5][3], result[4][3])
self.assertEqual(result[5][4], taustart_tss[2])
def test_nullDetection(self):
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, each with 4 bands -> 12 images.
taustart_tss = [
datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)
]
freq_effs = [124, 149, 156, 185]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.generate_timespaced_dbimages_data(
len(freq_effs) * len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(
im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image.
src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images), ) * len(freq_effs)):
for image in images:
# The first source is only seen at timestep 0, band 0.
# The second source is only seen at timestep 1, band 3.
if (image.taustart_ts == taustart_tss[0]
and image.freq_eff == freq_effs[0]):
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
elif (image.taustart_ts == taustart_tss[1]
and image.freq_eff == freq_effs[3]):
dbgen.insert_extracted_sources(image.id, [src1], 'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id,
deRuiter_r=5.68,
new_source_sigma_margin=3)
nd_ids_pos = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(nd_ids_pos), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 is a null detection at the second timestep
self.assertEqual(len(nd_ids_pos), 1)
dbgen.insert_extracted_sources(
image.id, [src0],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
else:
# All other images have two null detections.
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(
image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have eight runningcatalog_flux entries,
# one for every source in every band, i.e. 2 x 4.
# The number of flux datapoints differ per source, though
self.assertEqual(len(result), 8)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in band 0 images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in bands 1,2,3 images at next timesteps.
# so 0+2 for bands 1,2,3.
self.assertEqual(result[1][2], 2)
self.assertEqual(result[2][2], 2)
self.assertEqual(result[3][2], 2)
# Source 2: inserted into timestep 1, band 3.
# Force-fits in band 0,1,2 images at next timestep,
# so 1 for band 0,1,2
self.assertEqual(result[4][2], 1)
self.assertEqual(result[5][2], 1)
self.assertEqual(result[6][2], 1)
# Source 2: inserted into timestep 1, band 3.
# Force-fit in band 3 image at next timestep,
# so 1+1 for band 3
self.assertEqual(result[7][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3)
# and 2 datapoints for the other three bands (t2,t3).
# Source 2 has two datapoints for band3 (t2,t3) and
# one for the other three bands (t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 9 + 5 entries for source 1 and 2 resp.
self.assertEqual(len(result), 14)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source1: Forced fit at t2, band1
self.assertEqual(result[3][2], 7)
self.assertTrue(result[3][3] > result[2][3])
self.assertEqual(result[3][4], taustart_tss[1])
# Source1: Forced fit at t3, band1
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[2])
# Source1: Forced fit at t2, band2
self.assertEqual(result[5][2], 7)
self.assertTrue(result[5][3] > result[4][3])
self.assertEqual(result[5][4], taustart_tss[1])
# Source1: Forced fit at t3, band2
self.assertEqual(result[6][2], 7)
self.assertEqual(result[6][3], result[5][3])
self.assertEqual(result[6][4], taustart_tss[2])
# Source1: Forced fit at t2, band3
self.assertEqual(result[7][2], 7)
self.assertTrue(result[7][3] > result[6][3])
self.assertEqual(result[7][4], taustart_tss[1])
# Source1: Forced fit at t3, band3
self.assertEqual(result[8][2], 7)
self.assertEqual(result[8][3], result[7][3])
self.assertEqual(result[8][4], taustart_tss[2])
# Source2: Forced fit at t3, band0
self.assertEqual(result[9][2], 7)
self.assertEqual(result[9][3], result[0][3])
self.assertEqual(result[9][4], taustart_tss[2])
# Source2: Forced fit at t3, band1
self.assertEqual(result[10][2], 7)
self.assertTrue(result[10][3] > result[9][3])
self.assertEqual(result[10][4], taustart_tss[2])
# Source2: Forced fit at t3, band2
self.assertEqual(result[11][2], 7)
self.assertTrue(result[11][3] > result[10][3])
self.assertEqual(result[11][4], taustart_tss[2])
# Source2: new at t2, band3
self.assertEqual(result[12][2], 4)
self.assertTrue(result[12][3] > result[11][3])
self.assertEqual(result[12][4], taustart_tss[1])
# Source2: Forced fit at t3, band3
self.assertEqual(result[13][2], 7)
self.assertEqual(result[13][3], result[12][3])
self.assertEqual(result[13][4], taustart_tss[2])
def test_1to1_nullDetection(self):
"""
This tests that the two sources are associated if they were
detected at different timesteps. The positions are used in
the next test as well.
"""
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Two timesteps, just 1 band -> 2 images.
taustart_tss = [
datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1)
]
freq_effs = [124]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.generate_timespaced_dbimages_data(
len(freq_effs) * len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(
im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image
# and close together (see next test)
src0 = db_subs.example_extractedsource_tuple(ra=122.985, dec=10.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.015, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images), ) * len(freq_effs)):
for image in images:
# The sources are only seen at timestep 0
if (image.taustart_ts == taustart_tss[0]):
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
elif (image.taustart_ts == taustart_tss[1]):
dbgen.insert_extracted_sources(image.id, [src1], 'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id,
deRuiter_r=5.68,
new_source_sigma_margin=3)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have one runningcatalog sources, with two datapoints
# for the images in which the sources were seen.
self.assertEqual(len(result), 1)
self.assertEqual(result[0][1], 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have one runningcatalog_flux entry,
# where the source has two flux datapoints
self.assertEqual(len(result), 1)
self.assertEqual(result[0][2], 2)
def test_only_first_epoch_source(self):
"""test_only_first_epoch_source
- Pretend to extract a source only from the first image.
- Run source association for each image, as we would in TraP.
- Check the image source listing works
- Check runcat and assocxtrsource are correct.
"""
first_epoch = True
extracted_source_ids = []
for im in self.im_params:
self.db_imgs.append(Image(data=im, dataset=self.dataset))
last_img = self.db_imgs[-1]
if first_epoch:
insert_extracted_sources(
last_img._id, [db_subs.example_extractedsource_tuple()],
'blind')
associate_extracted_sources(last_img._id, deRuiter_r,
new_source_sigma_margin)
# First, check the runcat has been updated correctly
running_cat = columns_from_table(
table="runningcatalog",
keywords=['datapoints'],
where={"dataset": self.dataset.id})
self.assertEqual(len(running_cat), 1)
self.assertEqual(running_cat[0]['datapoints'], 1)
last_img.update()
last_img.update_sources()
img_xtrsrc_ids = [src.id for src in last_img.sources]
if first_epoch:
self.assertEqual(len(img_xtrsrc_ids), 1)
extracted_source_ids.extend(img_xtrsrc_ids)
assocxtrsrcs_rows = columns_from_table(
table="assocxtrsource",
keywords=['runcat', 'xtrsrc'],
where={"xtrsrc": img_xtrsrc_ids[0]})
self.assertEqual(len(assocxtrsrcs_rows), 1)
self.assertEqual(assocxtrsrcs_rows[0]['xtrsrc'],
img_xtrsrc_ids[0])
else:
self.assertEqual(len(img_xtrsrc_ids), 0)
first_epoch = False
#Assocxtrsources still ok after multiple images?
self.assertEqual(len(extracted_source_ids), 1)
assocxtrsrcs_rows = columns_from_table(
table="assocxtrsource",
keywords=['runcat', 'xtrsrc'],
where={"xtrsrc": extracted_source_ids[0]})
self.assertEqual(len(assocxtrsrcs_rows), 1)
self.assertEqual(
assocxtrsrcs_rows[0]['xtrsrc'], extracted_source_ids[0],
"Runcat xtrsrc entry must match the only extracted source")
def test_one2manyflux(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-n'})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
central_ra, central_dec = 123.1235, 10.55,
position_offset_deg = 100. / 3600 #100 arcsec = 0.03 deg approx
# image 1
image = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[0])
imageid1 = image.id
img1_srclist = []
# 1 source
img1_srclist.append(
db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak=1.5,
peak_err=5e-1,
flux=3.0,
flux_err=5e-1,
))
dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind')
associate_extracted_sources(imageid1, deRuiter_r=3.717)
# image 2
image = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[1])
imageid2 = image.id
img2_srclist = []
# 2 sources (both close to source 1, catching the 1-to-many case)
img2_srclist.append(
db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak=1.6,
peak_err=5e-1,
flux=3.2,
flux_err=5e-1,
))
img2_srclist.append(
db_subs.example_extractedsource_tuple(
central_ra + position_offset_deg,
central_dec,
peak=1.9,
peak_err=5e-1,
flux=3.4,
flux_err=5e-1,
))
dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind')
associate_extracted_sources(imageid2, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img2_srclist[1] has larger RA value.
lightcurves_sorted_by_ra = []
lightcurves_sorted_by_ra.append([img1_srclist[0], img2_srclist[0]])
lightcurves_sorted_by_ra.append([img1_srclist[0], img2_srclist[1]])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), 2)
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset': dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database, rcid)
py_indices = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_many2manyflux_reduced_to_two_1to1(self):
"""
(See also assoc. test test_many2many_reduced_to_two_1to1 )
In this test-case we cross-associate between a rhombus of sources spread
about a central position, east-west in the first image,
north-south in the second.
The latter, north-south pair are slightly offset towards positive RA
and negative RA respectively.
The result is that the candidate associations are pruned down to
two one-to-one pairings..
"""
dataset = tkp.db.DataSet(database=self.database, data={'description': 'flux test set: n-m, ' + self._testMethodName})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
centre_ra, centre_dec = 123., 10.5,
offset_deg = 20 / 3600. #20 arcsec
tiny_offset_deg = 1 / 3600. #1 arcsec
eastern_src = db_subs.example_extractedsource_tuple(
ra=centre_ra + offset_deg,
dec=centre_dec,
peak = 1.5, peak_err = 1e-1,
flux = 3.0, flux_err = 1e-1,)
western_src = db_subs.example_extractedsource_tuple(
ra=centre_ra - offset_deg,
dec=centre_dec,
peak = 1.7, peak_err = 1e-1,
flux = 3.2, flux_err = 1e-1,)
northern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra + tiny_offset_deg,
dec=centre_dec + offset_deg,
peak = 1.8, peak_err = 1e-1,
flux = 3.3, flux_err = 1e-1,
)
southern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra - tiny_offset_deg,
dec=centre_dec - offset_deg,
peak = 1.4, peak_err = 1e-1,
flux = 2.9, flux_err = 1e-1,)
# image 1
image1 = tkp.db.Image(database=self.database, dataset=dataset,
data=im_params[0])
dbgen.insert_extracted_sources(
image1.id, [eastern_src,western_src], 'blind')
associate_extracted_sources(image1.id, deRuiter_r = 3.717)
# image 2
image2 = tkp.db.Image(database=self.database, dataset=dataset,
data=im_params[1])
dbgen.insert_extracted_sources(
image2.id, [northern_source, southern_source], 'blind')
associate_extracted_sources(image2.id, deRuiter_r = 3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img1_srclist[1] has larger RA value.
lightcurves_sorted_by_ra =[]
lightcurves_sorted_by_ra.append( [western_src, southern_source])
lightcurves_sorted_by_ra.append( [eastern_src, northern_source])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra, r.wm_decl
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':dataset.id},
order='wm_ra,wm_decl')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_single_fixed_source(self):
"""test_single_fixed_source
- Pretend to extract the same source in each of a series of images.
- Perform source association
- Check the image source listing works
- Check runcat, assocxtrsource.
"""
fixed_src_runcat_id = None
for img_idx, im in enumerate(self.im_params):
self.db_imgs.append(Image(data=im, dataset=self.dataset))
last_img = self.db_imgs[-1]
insert_extracted_sources(last_img._id,
[db_subs.example_extractedsource_tuple()],
'blind')
associate_extracted_sources(last_img._id, deRuiter_r,
new_source_sigma_margin)
running_cat = columns_from_table(
table="runningcatalog",
keywords=['id', 'datapoints'],
where={"dataset": self.dataset.id})
self.assertEqual(len(running_cat), 1)
self.assertEqual(running_cat[0]['datapoints'], img_idx + 1)
# Check runcat ID does not change for a steady single source
if img_idx == 0:
fixed_src_runcat_id = running_cat[0]['id']
self.assertIsNotNone(fixed_src_runcat_id,
"No runcat id assigned to source")
self.assertEqual(running_cat[0]['id'], fixed_src_runcat_id,
"Multiple runcat ids for same fixed source")
runcat_flux = columns_from_table(
table="runningcatalog_flux",
keywords=['f_datapoints'],
where={"runcat": fixed_src_runcat_id})
self.assertEqual(len(runcat_flux), 1)
self.assertEqual(img_idx + 1, runcat_flux[0]['f_datapoints'])
last_img.update()
last_img.update_sources()
img_xtrsrc_ids = [src.id for src in last_img.sources]
self.assertEqual(len(img_xtrsrc_ids), 1)
#Get the association row for most recent extraction:
assocxtrsrcs_rows = columns_from_table(
table="assocxtrsource",
keywords=['runcat', 'xtrsrc'],
where={"xtrsrc": img_xtrsrc_ids[0]})
# print "ImageID:", last_img.id
# print "Imgs sources:", img_xtrsrc_ids
# print "Assoc entries:", assocxtrsrcs_rows
# print "First extracted source id:", ds_source_ids[0]
# if len(assocxtrsrcs_rows):
# print "Associated source:", assocxtrsrcs_rows[0]['xtrsrc']
self.assertEqual(
len(assocxtrsrcs_rows),
1,
msg="No entries in assocxtrsrcs for image number " +
str(img_idx))
self.assertEqual(assocxtrsrcs_rows[0]['runcat'],
fixed_src_runcat_id,
"Mismatched runcat id in assocxtrsrc table")
def run(job_name, mon_coords, local=False):
setup_event_listening(celery_app)
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
metadatas = runner(tasks.persistence_node_step, imgs, [image_cache_params],
local)
metadatas = [m[0] for m in metadatas]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner(tasks.quality_reject_check, urls, arguments, local)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extract_sources = runner(tasks.extract_sources, urls, arguments, local)
logger.info("storing extracted to database")
for image, sources in zip(images, extract_sources):
dbgen.insert_extracted_sources(image.id, sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius)
logger.info("performing null detections")
null_detections = dbnd.get_nulldetections(image.id)
logger.info("Found %s null detections" % len(null_detections))
# Only if we found null_detections the next steps are necessary
if len(null_detections) > 0:
logger.info("performing forced fits")
ff_nd = forced_fits(image.url, null_detections, se_parset)
dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd')
logger.info("adding null detections")
dbnd.associate_nd(image.id)
if len(mon_coords) > 0:
logger.info("performing monitoringlist")
ff_ms = forced_fits(image.url, mon_coords, se_parset)
dbgen.insert_extracted_sources(image.id, ff_ms, 'ff_ms')
logger.info("adding monitoring sources")
dbmon.associate_ms(image.id)
transients = search_transients(image.id,
job_config['transient_search'])
dbgen.update_dataset_process_end_ts(dataset_id)
def test_lightsurface(self):
images = []
# make 4 * 5 images with different frequencies and date
for frequency in [80e6, 90e6, 100e6, 110e6, 120e6]:
for day in [3, 4, 5, 6]:
img_data = db_subs.example_dbimage_data_dict(
taustart_ts=datetime.datetime(2010, 3, day),
freq_eff=frequency)
image = Image(dataset=self.dataset, data=img_data)
images.append(image)
# 3 sources per image, with different coordinates & flux
data_list = []
for i in range(1, 4):
data_list.append({
'ra': 111.111 + i,
'decl': 11.11 + i,
'ra_fit_err': 0.01,
'decl_fit_err': 0.01,
'i_peak': 10. * i,
'i_peak_err': 0.1,
'error_radius': 10.0,
'fit_type': 1,
# x=0.11, y=0.22, z=0.33, det_sigma=11.1, zone=i
})
# Insert the 3 sources in each image, while further varying the flux
for i, image in enumerate(images):
# Create the "source finding results"
sources = []
for data in data_list:
source = db_subs.example_extractedsource_tuple(
ra=data['ra'],
dec=data['decl'],
ra_fit_err=data['ra_fit_err'],
dec_fit_err=data['decl_fit_err'],
peak=data['i_peak'] * (1 + i),
peak_err=data['i_peak_err'],
flux=data['i_peak'] * (1 + i),
flux_err=data['i_peak_err'],
fit_type=data['fit_type'])
sources.append(source)
# Insert the sources
insert_extracted_sources(image._id, sources)
# Run the association for each list of source for an image
associate_extracted_sources(image._id,
deRuiter_r=3.7,
new_source_sigma_margin=3)
# updates the dataset and its set of images
self.dataset.update()
self.dataset.update_images()
# update the images and their sets of sources
for image in self.dataset.images:
image.update()
image.update_sources()
# Now pick any image, select the first source (smallest RA)
# and extract its light curve
# TODO: aaarch this is so ugly. Because this a set we need to pop it.
sources = self.dataset.images.pop().sources
#sources = self.dataset.images[-1].sources
sources = sorted(sources, key=attrgetter('ra'))
extracted_source = sources[0].id
lightcurve = tkp.db.general.lightcurve(extracted_source)
def test_nullDetection(self):
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, each with 4 bands -> 12 images.
taustart_tss = [datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)]
freq_effs = [124, 149, 156, 185]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.example_dbimage_datasets(len(freq_effs)
* len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image.
src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images),) * len(freq_effs)):
for image in images:
# The first source is only seen at timestep 0, band 0.
# The second source is only seen at timestep 1, band 3.
if (image.taustart_ts == taustart_tss[0] and
image.freq_eff == freq_effs[0]):
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
elif (image.taustart_ts == taustart_tss[1] and
image.freq_eff == freq_effs[3]):
dbgen.insert_extracted_sources(image.id, [src1], 'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id, deRuiter_r=5.68)
null_detections = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(null_detections), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 is a null detection at the second timestep
self.assertEqual(len(null_detections), 1)
dbgen.insert_extracted_sources(image.id, [src0], 'ff_nd')
else:
# All other images have two null detections.
self.assertEqual(len(null_detections), 2)
dbgen.insert_extracted_sources(image.id, [src0, src1],
'ff_nd')
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have eight runningcatalog_flux entries,
# one for every source in every band, i.e. 2 x 4.
# The number of flux datapoints differ per source, though
self.assertEqual(len(result), 8)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in band 0 images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in bands 1,2,3 images at next timesteps.
# so 0+2 for bands 1,2,3.
self.assertEqual(result[1][2], 2)
self.assertEqual(result[2][2], 2)
self.assertEqual(result[3][2], 2)
# Source 2: inserted into timestep 1, band 3.
# Force-fits in band 0,1,2 images at next timestep,
# so 1 for band 0,1,2
self.assertEqual(result[4][2], 1)
self.assertEqual(result[5][2], 1)
self.assertEqual(result[6][2], 1)
# Source 2: inserted into timestep 1, band 3.
# Force-fit in band 3 image at next timestep,
# so 1+1 for band 3
self.assertEqual(result[7][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3)
# and 2 datapoints for the other three bands (t2,t3).
# Source 2 has two datapoints for band3 (t2,t3) and
# one for the other three bands (t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 9 + 5 entries for source 1 and 2 resp.
self.assertEqual(len(result), 14)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source1: Forced fit at t2, band1
self.assertEqual(result[3][2], 7)
self.assertTrue(result[3][3] > result[2][3])
self.assertEqual(result[3][4], taustart_tss[1])
# Source1: Forced fit at t3, band1
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[2])
# Source1: Forced fit at t2, band2
self.assertEqual(result[5][2], 7)
self.assertTrue(result[5][3] > result[4][3])
self.assertEqual(result[5][4], taustart_tss[1])
# Source1: Forced fit at t3, band2
self.assertEqual(result[6][2], 7)
self.assertEqual(result[6][3], result[5][3])
self.assertEqual(result[6][4], taustart_tss[2])
# Source1: Forced fit at t2, band3
self.assertEqual(result[7][2], 7)
self.assertTrue(result[7][3] > result[6][3])
self.assertEqual(result[7][4], taustart_tss[1])
# Source1: Forced fit at t3, band3
self.assertEqual(result[8][2], 7)
self.assertEqual(result[8][3], result[7][3])
self.assertEqual(result[8][4], taustart_tss[2])
# Source2: Forced fit at t3, band0
self.assertEqual(result[9][2], 7)
self.assertEqual(result[9][3], result[0][3])
self.assertEqual(result[9][4], taustart_tss[2])
# Source2: Forced fit at t3, band1
self.assertEqual(result[10][2], 7)
self.assertTrue(result[10][3] > result[9][3])
self.assertEqual(result[10][4], taustart_tss[2])
# Source2: Forced fit at t3, band2
self.assertEqual(result[11][2], 7)
self.assertTrue(result[11][3] > result[10][3])
self.assertEqual(result[11][4], taustart_tss[2])
# Source2: new at t2, band3
self.assertEqual(result[12][2], 4)
self.assertTrue(result[12][3] > result[11][3])
self.assertEqual(result[12][4], taustart_tss[1])
# Source2: Forced fit at t3, band3
self.assertEqual(result[13][2], 7)
self.assertEqual(result[13][3], result[12][3])
self.assertEqual(result[13][4], taustart_tss[2])
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.)
#Sorted by increasing RA:
mon_srcs = [
superimposed_mon_src, mon_src_in_field, mon_src_out_of_field
]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id, mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests), 0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests), len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key=lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(
img.id, [superimposed_mon_src, mon_src_in_field],
'ff_ms',
ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(
self.dataset.database, runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [
blind_src_assocs, superimposed_mon_src_assocs,
offset_mon_src_assocs
]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'], 0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],
2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def insert_forcedfits_into_extractedsource(image_id, results, extract):
general.insert_extracted_sources(image_id, results, extract)
def test_one2manyflux(self):
dataset = tkp.db.DataSet(database=self.database,
data={'description': 'flux test set: 1-n'})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
central_ra, central_dec = 123.1235, 10.55,
position_offset_deg = 100./3600 #100 arcsec = 0.03 deg approx
# image 1
image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[0])
imageid1 = image.id
img1_srclist = []
# 1 source
img1_srclist.append(db_subs.example_extractedsource_tuple(central_ra, central_dec,
peak = 1.5, peak_err = 5e-1,
flux = 3.0, flux_err = 5e-1,
))
dbgen.insert_extracted_sources(imageid1, img1_srclist, 'blind')
associate_extracted_sources(imageid1, deRuiter_r=3.717)
# image 2
image = tkp.db.Image(database=self.database, dataset=dataset, data=im_params[1])
imageid2 = image.id
img2_srclist = []
# 2 sources (both close to source 1, catching the 1-to-many case)
img2_srclist.append(db_subs.example_extractedsource_tuple(
central_ra,
central_dec,
peak = 1.6, peak_err = 5e-1,
flux = 3.2, flux_err = 5e-1,
))
img2_srclist.append(db_subs.example_extractedsource_tuple(
central_ra + position_offset_deg,
central_dec,
peak = 1.9, peak_err = 5e-1,
flux = 3.4, flux_err = 5e-1,
))
dbgen.insert_extracted_sources(imageid2, img2_srclist, 'blind')
associate_extracted_sources(imageid2, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img2_srclist[1] has larger RA value.
lightcurves_sorted_by_ra =[]
lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[0]])
lightcurves_sorted_by_ra.append( [img1_srclist[0], img2_srclist[1]])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), 2)
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def run(job_name, local=False):
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
database_config(pipe_config)
job_dir = pipe_config.get('layout', 'job_directory')
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
images = imp.load_source('images_to_process', os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(images)))
job_config = load_job_config(pipe_config)
dump_job_config_to_logdir(pipe_config, job_config)
p_parset = parset.load_section(job_config, 'persistence')
se_parset = parset.load_section(job_config, 'source_extraction')
nd_parset = parset.load_section(job_config, 'null_detections')
tr_parset = parset.load_section(job_config, 'transient_search')
# persistence
imgs = [[img] for img in images]
arguments = [p_parset]
metadatas = runner(tasks.persistence_node_step, imgs, arguments, local)
metadatas = [m[0] for m in metadatas]
dataset_id, image_ids = steps.persistence.master_steps(metadatas,
se_parset['radius'],
p_parset)
# manual monitoringlist entries
if not add_manual_monitoringlist_entries(dataset_id, []):
return 1
images = [Image(id=image_id) for image_id in image_ids]
# quality_check
urls = [img.url for img in images]
arguments = [job_config]
rejecteds = runner(tasks.quality_reject_check, urls, arguments, local)
good_images = []
for image, rejected in zip(images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
# Sourcefinding
urls = [img.url for img in good_images]
arguments = [se_parset]
extract_sources = runner(tasks.extract_sources, urls, arguments, local)
for image, sources in zip(good_images, extract_sources):
dbgen.insert_extracted_sources(image.id, sources, 'blind')
# null_detections
deRuiter_radius = nd_parset['deruiter_radius']
null_detectionss = [dbmon.get_nulldetections(image.id, deRuiter_radius)
for image in good_images]
iters = zip([i.url for i in good_images], null_detectionss)
arguments = [nd_parset]
ff_nds = runner(tasks.forced_fits, iters, arguments, local)
for image, ff_nd in zip(good_images, ff_nds):
dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd')
for image in good_images:
logger.info("performing DB operations for image %s" % image.id)
dbass.associate_extracted_sources(image.id,
deRuiter_r=deRuiter_radius)
dbmon.add_nulldetections(image.id)
transients = steps.transient_search.search_transients(image.id,
tr_parset)
dbmon.adjust_transients_in_monitoringlist(image.id, transients)
for transient in transients:
steps.feature_extraction.extract_features(transient)
# ingred.classification.classify(transient, cl_parset)
now = datetime.datetime.utcnow()
dbgen.update_dataset_process_ts(dataset_id, now)
def test_many2manyflux_reduced_to_two_1to1(self):
"""
(See also assoc. test test_many2many_reduced_to_two_1to1 )
In this test-case we cross-associate between a rhombus of sources spread
about a central position, east-west in the first image,
north-south in the second.
The latter, north-south pair are slightly offset towards positive RA
and negative RA respectively.
The result is that the candidate associations are pruned down to
two one-to-one pairings..
"""
dataset = tkp.db.DataSet(database=self.database,
data={
'description':
'flux test set: n-m, ' +
self._testMethodName
})
n_images = 2
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
centre_ra, centre_dec = 123., 10.5,
offset_deg = 20 / 3600. #20 arcsec
tiny_offset_deg = 1 / 3600. #1 arcsec
eastern_src = db_subs.example_extractedsource_tuple(
ra=centre_ra + offset_deg,
dec=centre_dec,
peak=1.5,
peak_err=1e-1,
flux=3.0,
flux_err=1e-1,
)
western_src = db_subs.example_extractedsource_tuple(
ra=centre_ra - offset_deg,
dec=centre_dec,
peak=1.7,
peak_err=1e-1,
flux=3.2,
flux_err=1e-1,
)
northern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra + tiny_offset_deg,
dec=centre_dec + offset_deg,
peak=1.8,
peak_err=1e-1,
flux=3.3,
flux_err=1e-1,
)
southern_source = db_subs.example_extractedsource_tuple(
ra=centre_ra - tiny_offset_deg,
dec=centre_dec - offset_deg,
peak=1.4,
peak_err=1e-1,
flux=2.9,
flux_err=1e-1,
)
# image 1
image1 = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[0])
dbgen.insert_extracted_sources(image1.id, [eastern_src, western_src],
'blind')
associate_extracted_sources(image1.id, deRuiter_r=3.717)
# image 2
image2 = tkp.db.Image(database=self.database,
dataset=dataset,
data=im_params[1])
dbgen.insert_extracted_sources(image2.id,
[northern_source, southern_source],
'blind')
associate_extracted_sources(image2.id, deRuiter_r=3.717)
# Manually compose the lists of sources we expect to see associated
# into runningcatalog entries:
# NB img1_srclist[1] has larger RA value.
lightcurves_sorted_by_ra = []
lightcurves_sorted_by_ra.append([western_src, southern_source])
lightcurves_sorted_by_ra.append([eastern_src, northern_source])
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra, r.wm_decl
"""
self.database.cursor.execute(query, {'dataset': dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries),
len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset': dataset.id},
order='wm_ra,wm_decl')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database, rcid)
py_indices = db_subs.lightcurve_metrics(
lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key])
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.)
#Sorted by increasing RA:
mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id,mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests),0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests),len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key = lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(img.id,
[superimposed_mon_src, mon_src_in_field], 'ff_ms',
ff_monitor_ids=[mon_requests[0][0],
mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [blind_src_assocs,
superimposed_mon_src_assocs,
offset_mon_src_assocs]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'],0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_lightcurve(self):
# make 4 images with different date
images = []
image_datasets = db_subs.generate_timespaced_dbimages_data(n_images=4,
taustart_ts= datetime.datetime(2010, 3, 3)
)
for dset in image_datasets:
image = Image(dataset=self.dataset, data=dset)
images.append(image)
# 3 sources per image, with different coordinates & flux
data_list = []
for i in range(1, 4):
data_list.append({
'ra': 111.11 + i,
'decl': 11.11 + i,
'i_peak': 10. * i ,
'i_peak_err': 0.1,
})
# Insert the 3 sources in each image, while further varying the flux
lightcurves_sorted_by_ra = [[],[],[]]
for im_idx, image in enumerate(images):
# Create the "source finding results"
# Note that we reuse 'i_peak' as both peak & integrated flux.
img_sources = []
for src_idx, data in enumerate(data_list):
src = db_subs.example_extractedsource_tuple(
ra = data['ra'],dec=data['decl'],
peak=data['i_peak']* (1 + im_idx),
flux = data['i_peak']* (1 + im_idx)
)
lightcurves_sorted_by_ra[src_idx].append(src)
img_sources.append(src)
insert_extracted_sources(image._id, img_sources)
associate_extracted_sources(image._id, deRuiter_r=3.7,
new_source_sigma_margin=3)
# updates the dataset and its set of images
self.dataset.update()
self.dataset.update_images()
# update the images and their sets of sources
for image in self.dataset.images:
image.update()
image.update_sources()
# Now pick last image, select the first source (smallest RA)
# and extract its light curve
sources = self.dataset.images[-1].sources
sources = sorted(sources, key=attrgetter('ra'))
lightcurve = ligtcurve_func(sources[0]._id)
# check if the sources are associated in all images
self.assertEqual(len(images), len(lightcurve))
self.assertEqual(lightcurve[0][0], datetime.datetime(2010, 3, 3, 0, 0))
self.assertEqual(lightcurve[1][0], datetime.datetime(2010, 3, 4, 0, 0))
self.assertEqual(lightcurve[2][0], datetime.datetime(2010, 3, 5, 0, 0))
self.assertEqual(lightcurve[3][0], datetime.datetime(2010, 3, 6, 0, 0))
self.assertAlmostEqual(lightcurve[0][2], 10.)
self.assertAlmostEqual(lightcurve[1][2], 20.)
self.assertAlmostEqual(lightcurve[2][2], 30.)
self.assertAlmostEqual(lightcurve[3][2], 40.)
#Check the summary statistics (avg flux, etc)
query = """\
SELECT rf.avg_f_int
,rf.avg_f_int_sq
,avg_weighted_f_int
,avg_f_int_weight
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset)s
AND r.id = rf.runcat
ORDER BY r.wm_ra
"""
self.database.cursor.execute(query, {'dataset': self.dataset.id})
runcat_flux_entries = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(runcat_flux_entries), len(lightcurves_sorted_by_ra))
for idx, flux_summary in enumerate(runcat_flux_entries):
py_results = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
for key in flux_summary.keys():
self.assertAlmostEqual(flux_summary[key], py_results[-1][key])
#Now check the per-timestep statistics (variability indices)
sorted_runcat_ids = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id},
order='wm_ra')
sorted_runcat_ids = [entry['id'] for entry in sorted_runcat_ids]
for idx, rcid in enumerate(sorted_runcat_ids):
db_indices = db_queries.get_assoc_entries(self.database,
rcid)
py_indices = db_subs.lightcurve_metrics(lightcurves_sorted_by_ra[idx])
self.assertEqual(len(db_indices), len(py_indices))
for nstep in range(len(db_indices)):
for key in ('v_int', 'eta_int', 'f_datapoints'):
self.assertAlmostEqual(db_indices[nstep][key],
py_indices[nstep][key],
places=5)
def test_new_runcat_insertion(self):
"""Here we test the association logic executed upon insertion of a
new runningcatalog source.
We add an empty image0, then proceed to image1,
which is partially overlapping.
We add one new overlapping source, and one source only in image1's skyrgn.
Then we check that the back-associations to image0 are correct.
"""
n_images = 6
im_params = db_subs.generate_timespaced_dbimages_data(n_images)
#We first create 2 overlapping images,
#one above the other in dec by 1.0*xtr_radius
idx = 0
image0 = tkp.db.Image(dataset=self.dataset, data=im_params[idx])
image0.update()
#Bump up the centre of img1 to higher declination
im_params[1]['centre_decl'] += im_params[1]['xtr_radius']
#We place one source half-way between the field centres (i.e. in both)
src_in_imgs_0_1 = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] - im_params[1]['xtr_radius'] * 0.5)
#And one source only in field 1
src_in_img_1_only = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + im_params[1]['xtr_radius'] * 0.5)
# First insert new sources in img1 and check association to parent field
# This is always asserted without calculation, for efficiency
image1 = tkp.db.Image(dataset=self.dataset, data=im_params[1])
insert_extracted_sources(image1._id,
[src_in_imgs_0_1, src_in_img_1_only])
associate_extracted_sources(image1._id, deRuiter_r,
new_source_sigma_margin)
image1.update()
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
#We now expect to see both runcat entries in the field of im1
im1_assocs = columns_from_table(
'assocskyrgn', where={'skyrgn': image1._data['skyrgn']})
self.assertEqual(len(im1_assocs), 2)
runcat_ids = [r['id'] for r in runcats]
for assoc in im1_assocs:
self.assertTrue(assoc['runcat'] in runcat_ids)
#The new sources are *also checked against previous regions*
#Only expect one in field of im0 ( the first source).
im0_assocs = columns_from_table(
'assocskyrgn', where={'skyrgn': image0._data['skyrgn']})
runcats_only_in_im0 = columns_from_table('runningcatalog',
where={
'dataset':
self.dataset.id,
'wm_decl': 15
})
self.assertEqual(len(im0_assocs), 1)
self.assertEqual(len(runcats_only_in_im0), 1)
self.assertEqual(im0_assocs[0]['runcat'], runcats_only_in_im0[0]['id'])
