def test_probably_not_a_transient(self):
"""
No source at 250MHz, but we detect a source at 50MHz.
Not necessarily a transient.
Should trivially ignore 250MHz data when looking at a new 50MHz source.
"""
img_params = self.img_params
img0 = img_params[0]
# This time around, we just manually exclude the steady src from
# the first image detections.
steady_low_freq_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve=defaultdict(lambda: self.always_detectable_flux))
# Insert first image, no sources.
tkp.db.Image(data=img_params[0], dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1], dataset=self.dataset)
xtr = steady_low_freq_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r,
self.new_source_sigma_margin)
transients = get_newsources_for_dataset(self.dataset.id)
# Should have no marked transients
self.assertEqual(len(transients), 0)
def test_probably_not_a_transient(self):
"""
No source at 250MHz, but we detect a source at 50MHz.
Not necessarily a transient.
Should trivially ignore 250MHz data when looking at a new 50MHz source.
"""
img_params = self.img_params
img0 = img_params[0]
# This time around, we just manually exclude the steady src from
# the first image detections.
steady_low_freq_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']
),
lightcurve=defaultdict(lambda :self.always_detectable_flux)
)
# Insert first image, no sources.
tkp.db.Image(data=img_params[0],dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1],dataset=self.dataset)
xtr = steady_low_freq_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r, self.new_source_sigma_margin)
transients = get_newsources_for_dataset(self.dataset.id)
# Should have no marked transients
self.assertEqual(len(transients), 0)
def test_marginal_transient(self):
"""
( flux1 > (rms_min0*(det0 + margin) )
but ( flux1 < (rms_max0*(det0 + margin) )
--> Possible transient
If it was in a region of rms_min, we would (almost certainly) have seen
it in the first image. So new source --> Possible transient.
But if it was in a region of rms_max, then perhaps we would have missed
it. In which case, new source --> Just seeing deeper.
Note that if we are tiling overlapping images, then the first time
a field is processed with image-centre at the edge of the old field,
we may get a bunch of unhelpful 'possible transients'.
Furthermore, this will pick up fluctuating sources near the
image-margins even with a fixed field of view.
But without a more complex store of image-rms-per-position, we cannot
do better.
Hopefully we can use a 'distance from centre' feature to separate out
the good and bad candidates in this case.
"""
img_params = self.img_params
#Must pick flux value carefully to fire correct logic branch:
marginal_transient_flux = self.reliably_detected_at_image_centre_flux
marginal_transient = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve={img_params[1]['taustart_ts'] : marginal_transient_flux}
)
#First, check that we've set up the test correctly:
rms_min0 = img_params[0]['rms_min']
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_transient_flux <
rms_max0*(det0 + self.new_source_sigma_margin ) )
self.assertTrue(marginal_transient_flux >
rms_min0*(det0 + self.new_source_sigma_margin ) )
for pars in self.img_params:
img = tkp.db.Image(data=pars,dataset=self.dataset)
xtr = marginal_transient.simulate_extraction(img,
extraction_type='blind')
if xtr is not None:
img.insert_extracted_sources([xtr],'blind')
img.associate_extracted_sources(deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
#Should have one 'possible' transient
self.assertEqual(len(newsources),1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] < self.new_source_sigma_margin)
def test_null_detection_business_as_usual(self):
"""
If we do not blindly extract a steady source due to increased RMS,
then we expect a null-detection forced-fit to be triggered.
However, if the source properties are steady, this should not
result in the source being identified as transient.
"""
img0 = self.img_params[0]
steady_src_flux = self.barely_detectable_flux
steady_src = MockSource(
example_extractedsource_tuple(ra=img0['centre_ra'],
dec=img0['centre_decl']),
lightcurve=defaultdict(lambda: steady_src_flux))
image, blind_xtr, forced_fits = insert_image_and_simulated_sources(
self.dataset, self.img_params[0], [steady_src],
self.new_source_sigma_margin)
self.assertEqual(len(blind_xtr), 1)
self.assertEqual(len(forced_fits), 0)
image, blind_xtr, forced_fits = insert_image_and_simulated_sources(
self.dataset, self.img_params[1], [steady_src],
self.new_source_sigma_margin)
self.assertEqual(len(blind_xtr), 0)
self.assertEqual(len(forced_fits), 1)
get_sources_filtered_by_final_variability(dataset_id=self.dataset.id,
**self.search_params)
transients = get_newsources_for_dataset(self.dataset.id)
self.assertEqual(len(transients), 0)
def test_probably_not_a_transient(self):
"""
( flux1 < (rms_min0*(det0 + margin) )
--> Probably not a transient
NB even if
avg_source_flux == rms_min0*det0 + epsilon
we might not detect it in the
first image, due to noise fluctuations. So we provide the
user-tunable marginal_detection_thresh, to ignore these 'noise'
transients.
"""
img_params = self.img_params
img0 = img_params[0]
marginal_steady_src_flux = self.barely_detectable_flux
# This time around, we just manually exclude the steady src from
# the first image detections.
marginal_steady_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']
),
lightcurve=defaultdict(lambda :marginal_steady_src_flux)
)
# First, check that we've set up the test correctly
rms_min0 = img_params[0]['rms_min']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_steady_src_flux <
rms_min0 * (det0 + self.new_source_sigma_margin))
# Insert first image, no sources.
tkp.db.Image(data=img_params[0], dataset=self.dataset)
# Now set up second image.
img1 = tkp.db.Image(data=img_params[1], dataset=self.dataset)
xtr = marginal_steady_src.simulate_extraction(img1,
extraction_type='blind')
insert_extracted_sources(img1._id, [xtr], 'blind')
associate_extracted_sources(img1._id, deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
# Should have no flagged new sources
self.assertEqual(len(newsources), 0)
def test_probably_not_a_transient(self):
"""
( flux1 < (rms_min0*(det0 + margin) )
--> Probably not a transient
NB even if
avg_source_flux == rms_min0*det0 + epsilon
we might not detect it in the
first image, due to noise fluctuations. So we provide the
user-tunable marginal_detection_thresh, to ignore these 'noise'
transients.
"""
img_params = self.img_params
img0 = img_params[0]
marginal_steady_src_flux = self.barely_detectable_flux
# This time around, we just manually exclude the steady src from
# the first image detections.
marginal_steady_src = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']
),
lightcurve=defaultdict(lambda :marginal_steady_src_flux)
)
#First, check that we've set up the test correctly:
rms_min0 = img_params[0]['rms_min']
det0 = img_params[0]['detection_thresh']
self.assertTrue(marginal_steady_src_flux <
rms_min0*(det0 + self.new_source_sigma_margin ) )
#Insert first image, no sources.
tkp.db.Image(data=img_params[0],dataset=self.dataset)
#Now set up second image.
img1 = tkp.db.Image(data=img_params[1],dataset=self.dataset)
xtr = marginal_steady_src.simulate_extraction(img1,
extraction_type='blind')
img1.insert_extracted_sources([xtr],'blind')
img1.associate_extracted_sources(deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
#Should have no flagged new sources
self.assertEqual(len(newsources),0)
def test_certain_transient(self):
"""
flux1 > (rms_max0*(det0+margin)
--> Definite transient
Nice and bright, must be new - mark it definite transient.
"""
img_params = self.img_params
bright_transient = MockSource(example_extractedsource_tuple(
ra=img_params[0]['centre_ra'], dec=img_params[0]['centre_decl']),
lightcurve={
img_params[1]['taustart_ts']:
self.always_detectable_flux
})
#First, check that we've set up the test correctly:
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(bright_transient.lightcurve.values()[0] > rms_max0 *
(det0 + self.new_source_sigma_margin))
for pars in self.img_params:
img = tkp.db.Image(data=pars, dataset=self.dataset)
xtr = bright_transient.simulate_extraction(img,
extraction_type='blind')
if xtr is not None:
insert_extracted_sources(img._id, [xtr], 'blind')
associate_extracted_sources(img._id, deRuiter_r,
self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
#Should have one 'definite' transient
self.assertEqual(len(newsources), 1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(newsources[0]['low_thresh_sigma'] > newsources[0]
['high_thresh_sigma'])
def test_certain_transient(self):
"""
flux1 > (rms_max0*(det0+margin)
--> Definite transient
Nice and bright, must be new - mark it definite transient.
"""
img_params = self.img_params
bright_transient = MockSource(
example_extractedsource_tuple(ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve={img_params[1]['taustart_ts']:
self.always_detectable_flux}
)
#First, check that we've set up the test correctly:
rms_max0 = img_params[0]['rms_max']
det0 = img_params[0]['detection_thresh']
self.assertTrue(bright_transient.lightcurve.values()[0] >
rms_max0*(det0 + self.new_source_sigma_margin ) )
for pars in self.img_params:
img = tkp.db.Image(data=pars,dataset=self.dataset)
xtr = bright_transient.simulate_extraction(img,
extraction_type='blind')
if xtr is not None:
insert_extracted_sources(img._id, [xtr], 'blind')
associate_extracted_sources(img._id, deRuiter_r, self.new_source_sigma_margin)
newsources = get_newsources_for_dataset(self.dataset.id)
#Should have one 'definite' transient
self.assertEqual(len(newsources),1)
self.assertTrue(
newsources[0]['low_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['high_thresh_sigma'] > self.new_source_sigma_margin)
self.assertTrue(
newsources[0]['low_thresh_sigma'] >
newsources[0]['high_thresh_sigma'])
def test_previous_image_id(self):
img_params = self.img_params
mock_sources = []
mock_sources.append(
MockSource(example_extractedsource_tuple(
ra=img_params[0]['centre_ra'],
dec=img_params[0]['centre_decl']),
lightcurve={
img_params[-1]['taustart_ts']:
self.always_detectable_flux
}))
mock_sources.append(
MockSource(example_extractedsource_tuple(
ra=img_params[0]['centre_ra'] + 1,
dec=img_params[0]['centre_decl']),
lightcurve={
img_params[-1]['taustart_ts']:
self.always_detectable_flux
}))
image_ids = {}
for img_idx in xrange(self.n_images):
image, _, _ = insert_image_and_simulated_sources(
self.dataset, self.img_params[img_idx], mock_sources,
self.new_source_sigma_margin)
image_ids[img_idx] = image.id
newsources = get_newsources_for_dataset(self.dataset.id)
self.assertEqual(len(newsources), len(mock_sources))
newsource_properties = newsources[0]
print "Image IDs:", image_ids
self.assertEqual(newsource_properties['previous_limits_image'],
image_ids[4])
def setUp(self):
self.database = tkp.db.Database()
self.dataset = tkp.db.DataSet(
data={'description': "Trans:" + self._testMethodName},
database=self.database)
self.n_images = 8
self.image_rms = 1e-3 # 1mJy
self.new_source_sigma_margin = 3
self.search_params = dict(
eta_min=1,
v_min=0.1,
# minpoints=1,
)
detection_thresh = 10
barely_detectable_flux = 1.01 * self.image_rms * (detection_thresh)
reliably_detectable_flux = (
1.01 * self.image_rms *
(detection_thresh + self.new_source_sigma_margin))
test_specific_img_params = dict(rms_qc=self.image_rms,
rms_min=self.image_rms,
rms_max=self.image_rms,
detection_thresh=detection_thresh)
self.img_params = db_subs.generate_timespaced_dbimages_data(
self.n_images, **test_specific_img_params)
imgs = self.img_params
first_img = imgs[0]
centre_ra = first_img['centre_ra']
centre_decl = first_img['centre_decl']
xtr_radius = first_img['xtr_radius']
#At centre
fixed_source = MockSource(
example_extractedsource_tuple(ra=centre_ra, dec=centre_decl),
lightcurve=defaultdict(lambda: barely_detectable_flux))
#How many transients should we know about after each image?
self.n_transients_after_image = defaultdict(lambda: 0)
self.n_newsources_after_image = defaultdict(lambda: 0)
#shifted to +ve RA
bright_fast_transient = MockSource(
example_extractedsource_tuple(ra=centre_ra + xtr_radius * 0.5,
dec=centre_decl),
lightcurve={imgs[3]['taustart_ts']: reliably_detectable_flux})
#Detect immediately
for img_idx in range(3, self.n_images):
self.n_newsources_after_image[img_idx] += 1
#But only variable after non-detection
for img_idx in range(4, self.n_images):
self.n_transients_after_image[img_idx] += 1
# shifted to -ve RA
weak_fast_transient = MockSource(
example_extractedsource_tuple(ra=centre_ra - xtr_radius * 0.5,
dec=centre_decl),
lightcurve={imgs[3]['taustart_ts']: barely_detectable_flux})
# Not flagged as a newsource, could just be a weakly detected
# steady-source at first.
# But, shows high-variance after forced-fit in image[4]
for img_idx in range(4, self.n_images):
self.n_transients_after_image[img_idx] += 1
# shifted to +ve Dec
weak_slow_transient = MockSource(example_extractedsource_tuple(
ra=centre_ra, dec=centre_decl + xtr_radius * 0.5),
lightcurve={
imgs[5]['taustart_ts']:
barely_detectable_flux,
imgs[6]['taustart_ts']:
barely_detectable_flux * 0.95
})
# Not flagged as a newsource, could just be a weakly detected
# steady-source at first.
# Should not be flagged as transient until forced-fit in image[7]
for img_idx in range(7, self.n_images):
self.n_transients_after_image[img_idx] += 1
self.all_mock_sources = [
fixed_source, weak_slow_transient, bright_fast_transient,
weak_fast_transient
]
