def test_two_field_overlap_new_transient(self):
"""Now for something more interesting - two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at 2nd timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
imgs[0].insert_extracted_sources(
[lower_steady_src, overlap_steady_src])
imgs[0].associate_extracted_sources(
deRuiter_r=0.1, new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
imgs[1].insert_extracted_sources(
[upper_steady_src, overlap_steady_src, overlap_transient])
imgs[1].associate_extracted_sources(
deRuiter_r=0.1, new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 0)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id, ))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 1)
def test_two_field_overlap_new_transient(self):
"""Now for something more interesting - two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at 2nd timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
imgs[0].insert_extracted_sources([lower_steady_src, overlap_steady_src])
imgs[0].associate_extracted_sources(deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
imgs[1].insert_extracted_sources([upper_steady_src, overlap_steady_src,
overlap_transient])
imgs[1].associate_extracted_sources(deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 0)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
newsources_qry = """\
SELECT *
FROM newsource tr
,runningcatalog rc
WHERE rc.dataset = %s
AND tr.runcat = rc.id
"""
self.database.cursor.execute(newsources_qry, (self.dataset.id,))
newsources = get_db_rows_as_dicts(self.database.cursor)
self.assertEqual(len(newsources), 1)
def test_two_field_overlap_nulling_src(self):
"""Similar to above, but one source disappears:
Two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at *1st* timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(
n_images, xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
insert_extracted_sources(
imgs[0]._id,
[lower_steady_src, overlap_steady_src, overlap_transient])
associate_extracted_sources(
imgs[0]._id,
deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
insert_extracted_sources(imgs[1]._id,
[upper_steady_src, overlap_steady_src])
associate_extracted_sources(
imgs[1]._id,
deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
#This time we don't expect to get an immediate transient detection,
#but we *do* expect to get a null-source forced extraction request:
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 1)
runcats = columns_from_table('runningcatalog',
where={'dataset': self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
def test_two_field_overlap_nulling_src(self):
"""Similar to above, but one source disappears:
Two overlapping fields, 4 sources:
one steady source only in lower field,
one steady source in both fields,
one steady source only in upper field,
one transient source in both fields but only at *1st* timestep.
"""
n_images = 2
xtr_radius = 1.5
im_params = db_subs.generate_timespaced_dbimages_data(n_images,
xtr_radius=xtr_radius)
im_params[1]['centre_decl'] += xtr_radius * 1
imgs = []
lower_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] - 0.5 * xtr_radius)
upper_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[1]['centre_ra'],
dec=im_params[1]['centre_decl'] + 0.5 * xtr_radius)
overlap_steady_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.2 * xtr_radius)
overlap_transient = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'] + 0.8 * xtr_radius)
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[0]))
imgs.append(tkp.db.Image(dataset=self.dataset, data=im_params[1]))
imgs[0].insert_extracted_sources([lower_steady_src, overlap_steady_src,
overlap_transient])
imgs[0].associate_extracted_sources(deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
nd_posns = dbnd.get_nulldetections(imgs[0].id)
self.assertEqual(len(nd_posns), 0)
imgs[1].insert_extracted_sources([upper_steady_src, overlap_steady_src])
imgs[1].associate_extracted_sources(deRuiter_r=0.1,
new_source_sigma_margin=new_source_sigma_margin)
#This time we don't expect to get an immediate transient detection,
#but we *do* expect to get a null-source forced extraction request:
nd_posns = dbnd.get_nulldetections(imgs[1].id)
self.assertEqual(len(nd_posns), 1)
runcats = columns_from_table('runningcatalog',
where={'dataset':self.dataset.id})
self.assertEqual(len(runcats), 4) #sanity check.
def insert_image_and_simulated_sources(dataset,
image_params,
mock_sources,
new_source_sigma_margin,
deruiter_radius=3.7):
"""
Simulates the standard database image-and-source insertion logic using mock
sources.
Args:
dataset: The dataset object
image_params (dict): Contains the image properties.
mock_sources (list of MockSource): The mock sources to simulate.
new_source_sigma_margin (float): Parameter passed to source-association
routines.
deruiter_radius (float): Parameter passed to source-association
routines.
Returns:
3-tuple (image, list of blind extractions, list of forced fits).
"""
image = tkp.db.Image(data=image_params, dataset=dataset)
blind_extractions = []
for src in mock_sources:
xtr = src.simulate_extraction(image, extraction_type='blind')
if xtr is not None:
blind_extractions.append(xtr)
insert_extracted_sources(image._id, blind_extractions, 'blind')
associate_extracted_sources(
image._id,
deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_source_sigma_margin)
nd_ids_posns = nulldetections.get_nulldetections(image.id)
nd_posns = [(ra, decl) for ids, ra, decl in nd_ids_posns]
forced_fits = []
for posn in nd_posns:
for src in mock_sources:
eps = 1e-13
if (math.fabs(posn[0] - src.base_source.ra) < eps
and math.fabs(posn[1] - src.base_source.dec) < eps):
forced_fits.append(
src.simulate_extraction(image, extraction_type='ff_nd'))
if len(nd_posns) != len(forced_fits):
raise LookupError("Something went wrong, nulldetection position did "
"not match a mock source.")
#image.insert_extracted_sources(forced_fits, 'ff_nd')
dbgen.insert_extracted_sources(
image.id,
forced_fits,
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_posns])
nulldetections.associate_nd(image.id)
return image, blind_extractions, forced_fits
def test_nulldetections(self):
"""
Check if get_nulldetections doesn't return expired runcats
"""
# get all runningcatalog entries, should be two
runcats = self.session.query(Runningcatalog). \
filter(Runningcatalog.dataset == self.dataset).all()
self.assertEqual(len(runcats), 2)
# get_nulldetections before setting forcefits_count, should be 2
r = get_nulldetections(self.empty_image.id)
self.assertEqual(len(r), 2)
# set the forcedfits_count for one of the ligthcurves
runcats[0].forcedfits_count = 20
self.session.commit()
# fetch nulldetections, should be 1 now
r = get_nulldetections(self.empty_image.id)
self.assertEqual(len(r), 1)
def test_nulldetections(self):
"""
Check if get_nulldetections doesn't return expired runcats
"""
# get all runningcatalog entries, should be two
runcats = self.session.query(Runningcatalog). \
filter(Runningcatalog.dataset == self.dataset).all()
self.assertEqual(len(runcats), 2)
# get_nulldetections before setting forcefits_count, should be 2
r = get_nulldetections(self.empty_image.id)
self.assertEqual(len(r), 2)
# set the forcedfits_count for one of the ligthcurves
runcats[0].forcedfits_count = 20
self.session.commit()
# fetch nulldetections, should be 1 now
r = get_nulldetections(self.empty_image.id)
self.assertEqual(len(r), 1)
def insert_image_and_simulated_sources(dataset, image_params, mock_sources,
new_source_sigma_margin,
deruiter_radius=3.7):
"""
Simulates the standard database image-and-source insertion logic using mock
sources.
Args:
dataset: The dataset object
image_params (dict): Contains the image properties.
mock_sources (list of MockSource): The mock sources to simulate.
new_source_sigma_margin (float): Parameter passed to source-association
routines.
deruiter_radius (float): Parameter passed to source-association
routines.
Returns:
3-tuple (image, list of blind extractions, list of forced fits).
"""
image = tkp.db.Image(data=image_params,dataset=dataset)
blind_extractions=[]
for src in mock_sources:
xtr = src.simulate_extraction(image,extraction_type='blind')
if xtr is not None:
blind_extractions.append(xtr)
image.insert_extracted_sources(blind_extractions,'blind')
image.associate_extracted_sources(deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_source_sigma_margin)
nd_ids_posns = nulldetections.get_nulldetections(image.id)
nd_posns = [(ra,decl) for ids, ra, decl in nd_ids_posns]
forced_fits = []
for posn in nd_posns:
for src in mock_sources:
eps = 1e-13
if (math.fabs(posn[0] - src.base_source.ra)<eps and
math.fabs(posn[1] - src.base_source.dec)<eps ):
forced_fits.append(
src.simulate_extraction(image,extraction_type='ff_nd')
)
if len(nd_posns) != len(forced_fits):
raise LookupError("Something went wrong, nulldetection position did "
"not match a mock source.")
#image.insert_extracted_sources(forced_fits, 'ff_nd')
dbgen.insert_extracted_sources(image.id, forced_fits, 'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_posns])
nulldetections.associate_nd(image.id)
return image, blind_extractions, forced_fits
def get_forced_fit_requests(image, expiration):
nd_requested_fits = dbnd.get_nulldetections(image.id, expiration)
logger.debug("Found %s null detections" % len(nd_requested_fits))
mon_entries = dbmon.get_monitor_entries(image.dataset.id)
all_fit_positions = []
all_fit_ids = []
all_fit_positions.extend([(nd[1], nd[2]) for nd in nd_requested_fits])
all_fit_ids.extend([('ff_nd', nd[0]) for nd in nd_requested_fits])
all_fit_positions.extend([(ms[1], ms[2]) for ms in mon_entries])
all_fit_ids.extend([('ff_ms', ms[0]) for ms in mon_entries])
return all_fit_positions, all_fit_ids
def get_forced_fit_requests(image):
nd_requested_fits = dbnd.get_nulldetections(image.id)
logger.info("Found %s null detections" % len(nd_requested_fits))
mon_entries = dbmon.get_monitor_entries(image.dataset.id)
all_fit_positions = []
all_fit_ids = []
all_fit_positions.extend([(nd[1],nd[2]) for nd in nd_requested_fits])
all_fit_ids.extend([ ('ff_nd', nd[0]) for nd in nd_requested_fits])
all_fit_positions.extend([(ms[1],ms[2]) for ms in mon_entries])
all_fit_ids.extend([('ff_ms',ms[0]) for ms in mon_entries])
return all_fit_positions, all_fit_ids
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.)
#Sorted by increasing RA:
mon_srcs = [
superimposed_mon_src, mon_src_in_field, mon_src_out_of_field
]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id, mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests), 0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests), len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key=lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(
img.id, [superimposed_mon_src, mon_src_in_field],
'ff_ms',
ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(
self.dataset.database, runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [
blind_src_assocs, superimposed_mon_src_assocs,
offset_mon_src_assocs
]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'], 0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],
2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_m2m_nullDetection(self):
"""
This tests that two sources (close-by to be associated if they were
detected at different timesteps) which are not seen in the next
image and thus have forced fits, will have separate light curves.
The postions are from the previous test.
"""
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, just 1 band -> 3 images.
taustart_tss = [
datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)
]
freq_effs = [124]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.generate_timespaced_dbimages_data(
len(freq_effs) * len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(
im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image
# and close together (see previous test)
src0 = db_subs.example_extractedsource_tuple(ra=122.985, dec=10.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.015, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images), ) * len(freq_effs)):
for image in images:
# The sources are only seen at timestep 0
if (image.taustart_ts == taustart_tss[0]):
dbgen.insert_extracted_sources(image.id, [src0, src1],
'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id,
deRuiter_r=5.68,
new_source_sigma_margin=3)
nd_ids_pos = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(nd_ids_pos), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 & src1 are null detections at the second timestep
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(
image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
else:
# All other images have two null detections.
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(
image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog_flux entries,
# one for every source in the band, i.e. 2 x 1.
self.assertEqual(len(result), 2)
# Source 0: inserted into timestep 0.
# Force-fits in images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0
# Force-fits in images at next timesteps.
# so 1+2 for bands 0
self.assertEqual(result[1][2], 3)
#self.assertEqual(result[2][2], 2)
#self.assertEqual(result[3][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3).
# Source 2 also has 3 datapoints for band0 (t1,t2,t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 3 + 3 entries for source 0 and 1 resp.
self.assertEqual(len(result), 6)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source2: new at t1, band0
self.assertEqual(result[3][2], 4)
self.assertEqual(result[3][3], result[1][3])
self.assertEqual(result[3][4], taustart_tss[0])
# Source2: Forced fit at t2, band0
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[1])
# Source2: Forced fit at t3, band0
self.assertEqual(result[5][2], 7)
self.assertEqual(result[5][3], result[4][3])
self.assertEqual(result[5][4], taustart_tss[2])
def test_nullDetection(self):
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, each with 4 bands -> 12 images.
taustart_tss = [
datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)
]
freq_effs = [124, 149, 156, 185]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.generate_timespaced_dbimages_data(
len(freq_effs) * len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(
im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image.
src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images), ) * len(freq_effs)):
for image in images:
# The first source is only seen at timestep 0, band 0.
# The second source is only seen at timestep 1, band 3.
if (image.taustart_ts == taustart_tss[0]
and image.freq_eff == freq_effs[0]):
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
elif (image.taustart_ts == taustart_tss[1]
and image.freq_eff == freq_effs[3]):
dbgen.insert_extracted_sources(image.id, [src1], 'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id,
deRuiter_r=5.68,
new_source_sigma_margin=3)
nd_ids_pos = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(nd_ids_pos), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 is a null detection at the second timestep
self.assertEqual(len(nd_ids_pos), 1)
dbgen.insert_extracted_sources(
image.id, [src0],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
else:
# All other images have two null detections.
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(
image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have eight runningcatalog_flux entries,
# one for every source in every band, i.e. 2 x 4.
# The number of flux datapoints differ per source, though
self.assertEqual(len(result), 8)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in band 0 images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in bands 1,2,3 images at next timesteps.
# so 0+2 for bands 1,2,3.
self.assertEqual(result[1][2], 2)
self.assertEqual(result[2][2], 2)
self.assertEqual(result[3][2], 2)
# Source 2: inserted into timestep 1, band 3.
# Force-fits in band 0,1,2 images at next timestep,
# so 1 for band 0,1,2
self.assertEqual(result[4][2], 1)
self.assertEqual(result[5][2], 1)
self.assertEqual(result[6][2], 1)
# Source 2: inserted into timestep 1, band 3.
# Force-fit in band 3 image at next timestep,
# so 1+1 for band 3
self.assertEqual(result[7][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3)
# and 2 datapoints for the other three bands (t2,t3).
# Source 2 has two datapoints for band3 (t2,t3) and
# one for the other three bands (t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 9 + 5 entries for source 1 and 2 resp.
self.assertEqual(len(result), 14)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source1: Forced fit at t2, band1
self.assertEqual(result[3][2], 7)
self.assertTrue(result[3][3] > result[2][3])
self.assertEqual(result[3][4], taustart_tss[1])
# Source1: Forced fit at t3, band1
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[2])
# Source1: Forced fit at t2, band2
self.assertEqual(result[5][2], 7)
self.assertTrue(result[5][3] > result[4][3])
self.assertEqual(result[5][4], taustart_tss[1])
# Source1: Forced fit at t3, band2
self.assertEqual(result[6][2], 7)
self.assertEqual(result[6][3], result[5][3])
self.assertEqual(result[6][4], taustart_tss[2])
# Source1: Forced fit at t2, band3
self.assertEqual(result[7][2], 7)
self.assertTrue(result[7][3] > result[6][3])
self.assertEqual(result[7][4], taustart_tss[1])
# Source1: Forced fit at t3, band3
self.assertEqual(result[8][2], 7)
self.assertEqual(result[8][3], result[7][3])
self.assertEqual(result[8][4], taustart_tss[2])
# Source2: Forced fit at t3, band0
self.assertEqual(result[9][2], 7)
self.assertEqual(result[9][3], result[0][3])
self.assertEqual(result[9][4], taustart_tss[2])
# Source2: Forced fit at t3, band1
self.assertEqual(result[10][2], 7)
self.assertTrue(result[10][3] > result[9][3])
self.assertEqual(result[10][4], taustart_tss[2])
# Source2: Forced fit at t3, band2
self.assertEqual(result[11][2], 7)
self.assertTrue(result[11][3] > result[10][3])
self.assertEqual(result[11][4], taustart_tss[2])
# Source2: new at t2, band3
self.assertEqual(result[12][2], 4)
self.assertTrue(result[12][3] > result[11][3])
self.assertEqual(result[12][4], taustart_tss[1])
# Source2: Forced fit at t3, band3
self.assertEqual(result[13][2], 7)
self.assertEqual(result[13][3], result[12][3])
self.assertEqual(result[13][4], taustart_tss[2])
def test_nullDetection(self):
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, each with 4 bands -> 12 images.
taustart_tss = [datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)]
freq_effs = [124, 149, 156, 185]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.example_dbimage_datasets(len(freq_effs)
* len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image.
src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images),) * len(freq_effs)):
for image in images:
# The first source is only seen at timestep 0, band 0.
# The second source is only seen at timestep 1, band 3.
if (image.taustart_ts == taustart_tss[0] and
image.freq_eff == freq_effs[0]):
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
elif (image.taustart_ts == taustart_tss[1] and
image.freq_eff == freq_effs[3]):
dbgen.insert_extracted_sources(image.id, [src1], 'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id, deRuiter_r=5.68)
null_detections = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(null_detections), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 is a null detection at the second timestep
self.assertEqual(len(null_detections), 1)
dbgen.insert_extracted_sources(image.id, [src0], 'ff_nd')
else:
# All other images have two null detections.
self.assertEqual(len(null_detections), 2)
dbgen.insert_extracted_sources(image.id, [src0, src1],
'ff_nd')
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have eight runningcatalog_flux entries,
# one for every source in every band, i.e. 2 x 4.
# The number of flux datapoints differ per source, though
self.assertEqual(len(result), 8)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in band 0 images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0, band 0.
# Force-fits in bands 1,2,3 images at next timesteps.
# so 0+2 for bands 1,2,3.
self.assertEqual(result[1][2], 2)
self.assertEqual(result[2][2], 2)
self.assertEqual(result[3][2], 2)
# Source 2: inserted into timestep 1, band 3.
# Force-fits in band 0,1,2 images at next timestep,
# so 1 for band 0,1,2
self.assertEqual(result[4][2], 1)
self.assertEqual(result[5][2], 1)
self.assertEqual(result[6][2], 1)
# Source 2: inserted into timestep 1, band 3.
# Force-fit in band 3 image at next timestep,
# so 1+1 for band 3
self.assertEqual(result[7][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3)
# and 2 datapoints for the other three bands (t2,t3).
# Source 2 has two datapoints for band3 (t2,t3) and
# one for the other three bands (t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 9 + 5 entries for source 1 and 2 resp.
self.assertEqual(len(result), 14)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source1: Forced fit at t2, band1
self.assertEqual(result[3][2], 7)
self.assertTrue(result[3][3] > result[2][3])
self.assertEqual(result[3][4], taustart_tss[1])
# Source1: Forced fit at t3, band1
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[2])
# Source1: Forced fit at t2, band2
self.assertEqual(result[5][2], 7)
self.assertTrue(result[5][3] > result[4][3])
self.assertEqual(result[5][4], taustart_tss[1])
# Source1: Forced fit at t3, band2
self.assertEqual(result[6][2], 7)
self.assertEqual(result[6][3], result[5][3])
self.assertEqual(result[6][4], taustart_tss[2])
# Source1: Forced fit at t2, band3
self.assertEqual(result[7][2], 7)
self.assertTrue(result[7][3] > result[6][3])
self.assertEqual(result[7][4], taustart_tss[1])
# Source1: Forced fit at t3, band3
self.assertEqual(result[8][2], 7)
self.assertEqual(result[8][3], result[7][3])
self.assertEqual(result[8][4], taustart_tss[2])
# Source2: Forced fit at t3, band0
self.assertEqual(result[9][2], 7)
self.assertEqual(result[9][3], result[0][3])
self.assertEqual(result[9][4], taustart_tss[2])
# Source2: Forced fit at t3, band1
self.assertEqual(result[10][2], 7)
self.assertTrue(result[10][3] > result[9][3])
self.assertEqual(result[10][4], taustart_tss[2])
# Source2: Forced fit at t3, band2
self.assertEqual(result[11][2], 7)
self.assertTrue(result[11][3] > result[10][3])
self.assertEqual(result[11][4], taustart_tss[2])
# Source2: new at t2, band3
self.assertEqual(result[12][2], 4)
self.assertTrue(result[12][3] > result[11][3])
self.assertEqual(result[12][4], taustart_tss[1])
# Source2: Forced fit at t3, band3
self.assertEqual(result[13][2], 7)
self.assertEqual(result[13][3], result[12][3])
self.assertEqual(result[13][4], taustart_tss[2])
def run(job_name, mon_coords, local=False):
setup_event_listening(celery_app)
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
metadatas = runner(tasks.persistence_node_step, imgs, [image_cache_params],
local)
metadatas = [m[0] for m in metadatas]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner(tasks.quality_reject_check, urls, arguments, local)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extract_sources = runner(tasks.extract_sources, urls, arguments, local)
logger.info("storing extracted to database")
for image, sources in zip(images, extract_sources):
dbgen.insert_extracted_sources(image.id, sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius)
logger.info("performing null detections")
null_detections = dbnd.get_nulldetections(image.id)
logger.info("Found %s null detections" % len(null_detections))
# Only if we found null_detections the next steps are necessary
if len(null_detections) > 0:
logger.info("performing forced fits")
ff_nd = forced_fits(image.url, null_detections, se_parset)
dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd')
logger.info("adding null detections")
dbnd.associate_nd(image.id)
if len(mon_coords) > 0:
logger.info("performing monitoringlist")
ff_ms = forced_fits(image.url, mon_coords, se_parset)
dbgen.insert_extracted_sources(image.id, ff_ms, 'ff_ms')
logger.info("adding monitoring sources")
dbmon.associate_ms(image.id)
transients = search_transients(image.id,
job_config['transient_search'])
dbgen.update_dataset_process_end_ts(dataset_id)
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.)
#Sorted by increasing RA:
mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id,mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests),0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests),len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key = lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(img.id,
[superimposed_mon_src, mon_src_in_field], 'ff_ms',
ff_monitor_ids=[mon_requests[0][0],
mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [blind_src_assocs,
superimposed_mon_src_assocs,
offset_mon_src_assocs]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'],0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_m2m_nullDetection(self):
"""
This tests that two sources (close-by to be associated if they were
detected at different timesteps) which are not seen in the next
image and thus have forced fits, will have separate light curves.
The postions are from the previous test.
"""
data = {'description': "null detection:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, just 1 band -> 3 images.
taustart_tss = [datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)]
freq_effs = [124]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.generate_timespaced_dbimages_data(len(freq_effs)
* len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image
# and close together (see previous test)
src0 = db_subs.example_extractedsource_tuple(ra=122.985, dec=10.5)
src1 = db_subs.example_extractedsource_tuple(ra=123.015, dec=10.5)
# Group images in blocks of 4, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images),) * len(freq_effs)):
for image in images:
# The sources are only seen at timestep 0
if (image.taustart_ts == taustart_tss[0]):
dbgen.insert_extracted_sources(image.id, [src0,src1],
'blind')
else:
pass
for image in images:
dbass.associate_extracted_sources(image.id, deRuiter_r=5.68,
new_source_sigma_margin=3)
nd_ids_pos = dbnd.get_nulldetections(image.id)
# The null_detections are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0, src1
if image.taustart_ts == taustart_tss[0]:
# There are no null detections at the first timestep
self.assertEqual(len(nd_ids_pos), 0)
elif image.taustart_ts == taustart_tss[1]:
# src0 & src1 are null detections at the second timestep
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(image.id, [src0,src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
else:
# All other images have two null detections.
self.assertEqual(len(nd_ids_pos), 2)
dbgen.insert_extracted_sources(image.id, [src0, src1],
'ff_nd',
ff_runcat_ids=[ids for ids, ra, decl in nd_ids_pos])
# And here we have to associate the null detections with the
# runcat sources...
dbnd.associate_nd(image.id)
query = """\
SELECT id
,datapoints
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
ORDER BY datapoints
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, with a datapoint for
# every image in which the sources were seen.
self.assertEqual(len(result), 2)
query = """\
SELECT r.id
,rf.band
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
,rf.band
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog_flux entries,
# one for every source in the band, i.e. 2 x 1.
self.assertEqual(len(result), 2)
# Source 0: inserted into timestep 0.
# Force-fits in images at next timesteps,
# so 1+2 for band 0.
self.assertEqual(result[0][2], 3)
# Source 1: inserted into timestep 0
# Force-fits in images at next timesteps.
# so 1+2 for bands 0
self.assertEqual(result[1][2], 3)
#self.assertEqual(result[2][2], 2)
#self.assertEqual(result[3][2], 2)
# We should also have two lightcurves for both sources,
# where source 1 has 3 datapoints in band0 (t1,t2,t3).
# Source 2 also has 3 datapoints for band0 (t1,t2,t3).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.band
,i.taustart_ts
FROM assocxtrsource a
,extractedsource x
,image i
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
ORDER BY a.runcat
,i.band
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 3 + 3 entries for source 0 and 1 resp.
self.assertEqual(len(result), 6)
# The individual light-curve datapoints
# Source1: new at t1, band0
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][4], taustart_tss[0])
# Source1: Forced fit at t2, same band
self.assertEqual(result[1][2], 7)
self.assertEqual(result[1][3], result[0][3])
self.assertEqual(result[1][4], taustart_tss[1])
# Source1: Forced fit at t3, same band
self.assertEqual(result[2][2], 7)
self.assertEqual(result[2][3], result[1][3])
self.assertEqual(result[2][4], taustart_tss[2])
# Source2: new at t1, band0
self.assertEqual(result[3][2], 4)
self.assertEqual(result[3][3], result[1][3])
self.assertEqual(result[3][4], taustart_tss[0])
# Source2: Forced fit at t2, band0
self.assertEqual(result[4][2], 7)
self.assertEqual(result[4][3], result[3][3])
self.assertEqual(result[4][4], taustart_tss[1])
# Source2: Forced fit at t3, band0
self.assertEqual(result[5][2], 7)
self.assertEqual(result[5][3], result[4][3])
self.assertEqual(result[5][4], taustart_tss[2])