def insert_and_associate_forced_fits(image_id,successful_fits,successful_ids):
assert len(successful_ids) == len(successful_fits)
nd_extractions=[]
nd_runcats=[]
ms_extractions=[]
ms_ids = []
for idx, id in enumerate(successful_ids):
if id[0] == 'ff_nd':
nd_extractions.append(successful_fits[idx])
nd_runcats.append(id[1])
elif id[0] == 'ff_ms':
ms_extractions.append(successful_fits[idx])
ms_ids.append(id[1])
else:
raise ValueError("Forced fit type id not recognised:" + id[0])
if nd_extractions:
logger.debug("adding null detections")
dbgen.insert_extracted_sources(image_id, nd_extractions,
extract_type='ff_nd',
ff_runcat_ids=nd_runcats)
dbnd.associate_nd(image_id)
else:
logger.debug("No successful nulldetection fits")
if ms_extractions:
dbgen.insert_extracted_sources(image_id, ms_extractions,
extract_type='ff_ms',
ff_monitor_ids=ms_ids)
logger.debug("adding monitoring sources")
dbmon.associate_ms(image_id)
else:
logger.debug("No successful monitor fits")
def insert_and_associate_forced_fits(image_id, successful_fits,
successful_ids):
assert len(successful_ids) == len(successful_fits)
nd_extractions = []
nd_runcats = []
ms_extractions = []
ms_ids = []
for idx, id in enumerate(successful_ids):
if id[0] == 'ff_nd':
nd_extractions.append(successful_fits[idx])
nd_runcats.append(id[1])
elif id[0] == 'ff_ms':
ms_extractions.append(successful_fits[idx])
ms_ids.append(id[1])
else:
raise ValueError("Forced fit type id not recognised:" + id[0])
if nd_extractions:
logger.info("adding null detections")
dbgen.insert_extracted_sources(image_id,
nd_extractions,
extract_type='ff_nd',
ff_runcat_ids=nd_runcats)
dbnd.associate_nd(image_id)
else:
logger.info("No successful nulldetection fits")
if ms_extractions:
dbgen.insert_extracted_sources(image_id,
ms_extractions,
extract_type='ff_ms',
ff_monitor_ids=ms_ids)
logger.info("adding monitoring sources")
dbmon.associate_ms(image_id)
else:
logger.info("No successful monitor fits")
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.)
#Sorted by increasing RA:
mon_srcs = [
superimposed_mon_src, mon_src_in_field, mon_src_out_of_field
]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id, mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests), 0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests), len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key=lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(
img.id, [superimposed_mon_src, mon_src_in_field],
'ff_ms',
ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(
self.dataset.database, runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [
blind_src_assocs, superimposed_mon_src_assocs,
offset_mon_src_assocs
]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'], 0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],
2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.)
#Sorted by increasing RA:
mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id,mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests),0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests),len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key = lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(img.id,
[superimposed_mon_src, mon_src_in_field], 'ff_ms',
ff_monitor_ids=[mon_requests[0][0],
mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [blind_src_assocs,
superimposed_mon_src_assocs,
offset_mon_src_assocs]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'],0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def run(job_name, mon_coords, local=False):
setup_event_listening(celery_app)
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
metadatas = runner(tasks.persistence_node_step, imgs, [image_cache_params],
local)
metadatas = [m[0] for m in metadatas]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner(tasks.quality_reject_check, urls, arguments, local)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extract_sources = runner(tasks.extract_sources, urls, arguments, local)
logger.info("storing extracted to database")
for image, sources in zip(images, extract_sources):
dbgen.insert_extracted_sources(image.id, sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius)
logger.info("performing null detections")
null_detections = dbnd.get_nulldetections(image.id)
logger.info("Found %s null detections" % len(null_detections))
# Only if we found null_detections the next steps are necessary
if len(null_detections) > 0:
logger.info("performing forced fits")
ff_nd = forced_fits(image.url, null_detections, se_parset)
dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd')
logger.info("adding null detections")
dbnd.associate_nd(image.id)
if len(mon_coords) > 0:
logger.info("performing monitoringlist")
ff_ms = forced_fits(image.url, mon_coords, se_parset)
dbgen.insert_extracted_sources(image.id, ff_ms, 'ff_ms')
logger.info("adding monitoring sources")
dbmon.associate_ms(image.id)
transients = search_transients(image.id,
job_config['transient_search'])
dbgen.update_dataset_process_end_ts(dataset_id)
def test_monitoringSource(self):
data = {'description': "monitoringlist:" + self._testMethodName}
dataset = DataSet(data=data)
# Three timesteps, 1 band -> 3 images.
taustart_tss = [datetime.datetime(2013, 8, 1),
datetime.datetime(2013, 9, 1),
datetime.datetime(2013, 10, 1)]
#freq_effs = [124, 149, 156, 185]
freq_effs = [124]
freq_effs = [f * 1e6 for f in freq_effs]
im_params = db_subs.example_dbimage_datasets(len(freq_effs)
* len(taustart_tss))
timestamps = itertools.repeat(taustart_tss, len(freq_effs))
for im, freq, ts in zip(im_params, itertools.cycle(freq_effs),
itertools.chain.from_iterable(zip(*timestamps))):
im['freq_eff'] = freq
im['taustart_ts'] = ts
images = []
for im in im_params:
image = tkp.db.Image(dataset=dataset, data=im)
images.append(image)
# Arbitrary parameters, except that they fall inside our image.
# We have one to be monitored source and one "normal" source
src0 = db_subs.example_extractedsource_tuple(ra=122.5, dec=9.5)
src1_mon = db_subs.example_extractedsource_tuple(ra=123.5, dec=10.5)
# Group images in blocks of 1, corresponding to all frequency bands at
# a given timestep.
for images in zip(*(iter(images),) * len(freq_effs)):
for image in images:
# The "normal" source is seen at all timesteps
dbgen.insert_extracted_sources(image.id, [src0], 'blind')
for image in images:
dbass.associate_extracted_sources(image.id, deRuiter_r=5.68)
# The monitoring sources are the positional inputs for the forced
# fits, which on their turn return additional parameters,
# e.g. from src0_mon
# src1_mon is the monitoring source at all timesteps
dbgen.insert_extracted_sources(image.id, [src1_mon], 'ff_ms')
# And here we have to associate the monitoring sources with the
# runcat sources...
dbmon.associate_ms(image.id)
query = """\
SELECT id
,mon_src
FROM runningcatalog r
WHERE dataset = %(dataset_id)s
AND datapoints = 3
ORDER BY id
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog sources, one for the "normal"
# source and one for the monitoring source.
# Both should have three datapoints.
print "dp_result:",result
self.assertEqual(len(result), 2)
# The first source is the "normal" one
self.assertEqual(result[0][1], False)
# The second source is the monitoring one
self.assertEqual(result[1][1], True)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.id
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# We should have two runningcatalog_flux entries,
# one for every source, where every source has
# three f_datapoints
self.assertEqual(len(result), 2)
# "Normal" source: three flux datapoints
self.assertEqual(result[0][1], False)
self.assertEqual(result[0][2], 3)
# Monitoring source: three flux datapoints
self.assertEqual(result[1][1], True)
self.assertEqual(result[1][2], 3)
# We should also have two lightcurves for both sources,
# where both sources have three datapoints.
# The association types of the "normal" source are
# 3 (first) or 4 (later ones), while the monitoring source
# association types are 8 (first) or 9 (later ones).
query = """\
SELECT a.runcat
,a.xtrsrc
,a.type
,i.taustart_ts
,r.mon_src
,x.extract_type
FROM assocxtrsource a
,extractedsource x
,image i
,runningcatalog r
WHERE a.xtrsrc = x.id
AND x.image = i.id
AND i.dataset = %(dataset_id)s
AND a.runcat = r.id
ORDER BY a.runcat
,i.taustart_ts
"""
cursor = tkp.db.execute(query, {'dataset_id': dataset.id})
result = cursor.fetchall()
# 3 + 3 entries for source 1 and 2 resp.
self.assertEqual(len(result), 6)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(result[0][2], 4)
self.assertEqual(result[0][3], taustart_tss[0])
self.assertEqual(result[0][4], False)
self.assertEqual(result[0][5], 0)
# It was known at second timestep
self.assertEqual(result[1][2], 3)
self.assertEqual(result[1][3], taustart_tss[1])
self.assertEqual(result[1][4], result[0][4])
self.assertEqual(result[1][5], result[0][5])
# It was known at third timestep
self.assertEqual(result[2][2], result[1][2])
self.assertEqual(result[2][3], taustart_tss[2])
self.assertEqual(result[2][4], result[1][4])
self.assertEqual(result[2][5], result[1][5])
# The individual light-curve datapoints for the monitoring source
# It was new at first timestep
self.assertEqual(result[3][2], 8)
self.assertEqual(result[3][3], taustart_tss[0])
self.assertEqual(result[3][4], True)
self.assertEqual(result[3][5], 2)
# It was known at second timestep
self.assertEqual(result[4][2], 9)
self.assertEqual(result[4][3], taustart_tss[1])
self.assertEqual(result[4][4], result[3][4])
self.assertEqual(result[4][5], result[3][5])
# It was known at third timestep
self.assertEqual(result[5][2], result[4][2])
self.assertEqual(result[5][3], taustart_tss[2])
self.assertEqual(result[5][4], result[4][4])
self.assertEqual(result[5][5], result[4][5])