def initialise_dataset(job_config, supplied_mon_coords):
"""
sets up a dataset in the database.
if the dataset already exists it will return the job_config from the
previous dataset run.
args:
job_config: a job configuration object
supplied_mon_coords (list): a list of monitoring positions
returns:
tuple: job_config and dataset ID
"""
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
if supplied_mon_coords:
dbgen.insert_monitor_positions(dataset_id, supplied_mon_coords)
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
if supplied_mon_coords:
logger.warn("Monitor positions supplied will be ignored. "
"(Previous dataset specified)")
return job_config, dataset_id
def initialise_dataset(job_config, supplied_mon_coords):
"""
sets up a dataset in the database.
if the dataset already exists it will return the job_config from the
previous dataset run.
args:
job_config: a job configuration object
supplied_mon_coords (tuple): a list of monitoring positions
returns:
tuple: job_config and dataset ID
"""
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
if supplied_mon_coords:
dbgen.insert_monitor_positions(dataset_id, supplied_mon_coords)
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
if supplied_mon_coords:
logger.warn("Monitor positions supplied will be ignored. "
"(Previous dataset specified)")
return job_config, dataset_id
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra=blind_src.ra + 0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra=blind_src.ra + 90.)
#Sorted by increasing RA:
mon_srcs = [
superimposed_mon_src, mon_src_in_field, mon_src_out_of_field
]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id, mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests), 0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests), len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key=lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1], mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2], mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(
img.id, [superimposed_mon_src, mon_src_in_field],
'ff_ms',
ff_monitor_ids=[mon_requests[0][0], mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(
self.dataset.database, runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [
blind_src_assocs, superimposed_mon_src_assocs,
offset_mon_src_assocs
]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'], 0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],
2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'], 2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_insert(self):
dataset1 = DataSet(data=self.description)
monitor_positions = [(5., 5), (123, 85.)]
dbgen.insert_monitor_positions(dataset1.id, monitor_positions)
def run(job_name, supplied_mon_coords=[]):
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
# get parallelise props. Defaults to multiproc with autodetect num cores
parallelise = pipe_config.get('parallelise', {})
distributor = os.environ.get('TKP_PARALLELISE', parallelise.get('method',
'multiproc'))
runner = Runner(distributor=distributor,
cores=parallelise.get('cores', 0))
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
beamwidths_limit = job_config.association.beamwidths_limit
new_src_sigma = job_config.transient_search.new_source_sigma_margin
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
if supplied_mon_coords:
dbgen.insert_monitor_positions(dataset_id,supplied_mon_coords)
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
if supplied_mon_coords:
logger.warn("Monitor positions supplied will be ignored. "
"(Previous dataset specified)")
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
rms_est_sigma = job_config.persistence.rms_est_sigma
rms_est_fraction = job_config.persistence.rms_est_fraction
metadatas = runner.map("persistence_node_step", imgs,
[image_cache_params, rms_est_sigma, rms_est_fraction])
metadatas = [m[0] for m in metadatas if m]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner.map("quality_reject_check", urls, arguments)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extraction_results = runner.map("extract_sources", urls, arguments)
logger.info("storing extracted sources to database")
# we also set the image max,min RMS values which calculated during
# source extraction
for image, results in zip(images, extraction_results):
image.update(rms_min=results.rms_min, rms_max=results.rms_max,
detection_thresh=se_parset['detection_threshold'],
analysis_thresh=se_parset['analysis_threshold'])
dbgen.insert_extracted_sources(image.id, results.sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_src_sigma)
all_fit_posns, all_fit_ids = steps_ff.get_forced_fit_requests(image)
if all_fit_posns:
successful_fits, successful_ids = steps_ff.perform_forced_fits(
all_fit_posns, all_fit_ids, image.url, se_parset)
steps_ff.insert_and_associate_forced_fits(image.id,successful_fits,
successful_ids)
dbgen.update_dataset_process_end_ts(dataset_id)
def run(job_name, supplied_mon_coords=[]):
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
# get parallelise props. Defaults to multiproc with autodetect num cores
parallelise = pipe_config.get('parallelise', {})
distributor = os.environ.get('TKP_PARALLELISE', parallelise.get('method',
'multiproc'))
runner = Runner(distributor=distributor,
cores=parallelise.get('cores', 0))
debug = pipe_config.logging.debug
#Setup logfile before we do anything else
log_dir = pipe_config.logging.log_dir
setup_log_file(log_dir, debug)
job_dir = pipe_config.DEFAULT.job_directory
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
db_config = get_database_config(pipe_config.database, apply=True)
dump_database_backup(db_config, job_dir)
job_config = load_job_config(pipe_config)
se_parset = job_config.source_extraction
deruiter_radius = job_config.association.deruiter_radius
beamwidths_limit = job_config.association.beamwidths_limit
new_src_sigma = job_config.transient_search.new_source_sigma_margin
all_images = imp.load_source('images_to_process',
os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(all_images)))
logger.info("performing database consistency check")
if not dbconsistency.check():
logger.error("Inconsistent database found; aborting")
return 1
dataset_id = create_dataset(job_config.persistence.dataset_id,
job_config.persistence.description)
if job_config.persistence.dataset_id == -1:
store_config(job_config, dataset_id) # new data set
if supplied_mon_coords:
dbgen.insert_monitor_positions(dataset_id,supplied_mon_coords)
else:
job_config_from_db = fetch_config(dataset_id) # existing data set
if check_job_configs_match(job_config, job_config_from_db):
logger.debug("Job configs from file / database match OK.")
else:
logger.warn("Job config file has changed since dataset was "
"first loaded into database. ")
logger.warn("Using job config settings loaded from database, see "
"log dir for details")
job_config = job_config_from_db
if supplied_mon_coords:
logger.warn("Monitor positions supplied will be ignored. "
"(Previous dataset specified)")
dump_configs_to_logdir(log_dir, job_config, pipe_config)
logger.info("performing persistence step")
image_cache_params = pipe_config.image_cache
imgs = [[img] for img in all_images]
rms_est_sigma = job_config.persistence.rms_est_sigma
rms_est_fraction = job_config.persistence.rms_est_fraction
metadatas = runner.map("persistence_node_step", imgs,
[image_cache_params, rms_est_sigma, rms_est_fraction])
metadatas = [m[0] for m in metadatas if m]
logger.info("Storing images")
image_ids = store_images(metadatas,
job_config.source_extraction.extraction_radius_pix,
dataset_id)
db_images = [Image(id=image_id) for image_id in image_ids]
logger.info("performing quality check")
urls = [img.url for img in db_images]
arguments = [job_config]
rejecteds = runner.map("quality_reject_check", urls, arguments)
good_images = []
for image, rejected in zip(db_images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
grouped_images = group_per_timestep(good_images)
timestep_num = len(grouped_images)
for n, (timestep, images) in enumerate(grouped_images):
msg = "processing %s images in timestep %s (%s/%s)"
logger.info(msg % (len(images), timestep, n+1, timestep_num))
logger.info("performing source extraction")
urls = [img.url for img in images]
arguments = [se_parset]
extraction_results = runner.map("extract_sources", urls, arguments)
logger.info("storing extracted sources to database")
# we also set the image max,min RMS values which calculated during
# source extraction
for image, results in zip(images, extraction_results):
image.update(rms_min=results.rms_min, rms_max=results.rms_max,
detection_thresh=se_parset['detection_threshold'],
analysis_thresh=se_parset['analysis_threshold'])
dbgen.insert_extracted_sources(image.id, results.sources, 'blind')
logger.info("performing database operations")
for image in images:
logger.info("performing DB operations for image %s" % image.id)
logger.info("performing source association")
dbass.associate_extracted_sources(image.id,
deRuiter_r=deruiter_radius,
new_source_sigma_margin=new_src_sigma)
expiration = job_config.source_extraction.expiration
all_fit_posns, all_fit_ids = steps_ff.get_forced_fit_requests(image,
expiration)
if all_fit_posns:
successful_fits, successful_ids = steps_ff.perform_forced_fits(
all_fit_posns, all_fit_ids, image.url, se_parset)
steps_ff.insert_and_associate_forced_fits(image.id,successful_fits,
successful_ids)
dbgen.update_dataset_process_end_ts(dataset_id)
logger.info("calculating variability metrics")
execute_store_varmetric(dataset_id)
def test_basic_case(self):
im_params = self.im_params
blind_src = db_subs.example_extractedsource_tuple(
ra=im_params[0]['centre_ra'],
dec=im_params[0]['centre_decl'],
)
superimposed_mon_src = blind_src
mon_src_in_field = blind_src._replace(ra = blind_src.ra+0.001)
# Simulate a source that does not get fit, for good measure:
mon_src_out_of_field = blind_src._replace(ra = blind_src.ra+90.)
#Sorted by increasing RA:
mon_srcs = [superimposed_mon_src, mon_src_in_field, mon_src_out_of_field]
mon_posns = [(m.ra, m.dec) for m in mon_srcs]
dbgen.insert_monitor_positions(self.dataset.id,mon_posns)
images = []
for img_pars in self.im_params:
img = tkp.db.Image(dataset=self.dataset, data=img_pars)
dbgen.insert_extracted_sources(img.id, [blind_src], 'blind')
associate_extracted_sources(img.id, deRuiter_r=5.68)
nd_requests = get_nulldetections(img.id)
self.assertEqual(len(nd_requests),0)
mon_requests = dbmon.get_monitor_entries(self.dataset.id)
self.assertEqual(len(mon_requests),len(mon_srcs))
# mon requests is a list of tuples [(id,ra,decl)]
# Ensure sorted by RA for cross-checking:
mon_requests = sorted(mon_requests, key = lambda s: s[1])
for idx in range(len(mon_srcs)):
self.assertAlmostEqual(mon_requests[idx][1],mon_srcs[idx].ra)
self.assertAlmostEqual(mon_requests[idx][2],mon_srcs[idx].dec)
#Insert fits for the in-field sources and then associate
dbgen.insert_extracted_sources(img.id,
[superimposed_mon_src, mon_src_in_field], 'ff_ms',
ff_monitor_ids=[mon_requests[0][0],
mon_requests[1][0]])
dbmon.associate_ms(img.id)
query = """\
SELECT r.id
,r.mon_src
,rf.f_datapoints
FROM runningcatalog r
,runningcatalog_flux rf
WHERE r.dataset = %(dataset_id)s
AND rf.runcat = r.id
ORDER BY r.wm_ra
,r.mon_src
"""
cursor = tkp.db.execute(query, {'dataset_id': self.dataset.id})
runcat_flux = get_db_rows_as_dicts(cursor)
self.assertEqual(len(runcat_flux), 3)
# First entry (lowest RA, mon_src = False) is the regular one;
self.assertEqual(runcat_flux[0]['mon_src'], False)
# The higher RA source is the monitoring one
self.assertEqual(runcat_flux[1]['mon_src'], True)
self.assertEqual(runcat_flux[2]['mon_src'], True)
for entry in runcat_flux:
self.assertEqual(entry['f_datapoints'], len(self.im_params))
#Let's verify the association types
blind_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[0]['id'])
superimposed_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[1]['id'])
offset_mon_src_assocs = get_assoc_entries(self.dataset.database,
runcat_flux[2]['id'])
assoc_lists = [blind_src_assocs,
superimposed_mon_src_assocs,
offset_mon_src_assocs]
for al in assoc_lists:
self.assertEqual(len(al), 3)
# The individual light-curve datapoints for the "normal" source
# It was new at first timestep
self.assertEqual(blind_src_assocs[0]['type'], 4)
self.assertEqual(superimposed_mon_src_assocs[0]['type'], 8)
self.assertEqual(offset_mon_src_assocs[0]['type'], 8)
for idx, img_pars in enumerate(self.im_params):
if idx != 0:
self.assertEqual(blind_src_assocs[idx]['type'], 3)
self.assertEqual(superimposed_mon_src_assocs[idx]['type'], 9)
self.assertEqual(offset_mon_src_assocs[idx]['type'], 9)
#And the extraction types:
self.assertEqual(blind_src_assocs[idx]['extract_type'],0)
self.assertEqual(superimposed_mon_src_assocs[idx]['extract_type'],2)
self.assertEqual(offset_mon_src_assocs[idx]['extract_type'],2)
#Sanity check the timestamps while we're at it
for al in assoc_lists:
self.assertEqual(al[idx]['taustart_ts'],
img_pars['taustart_ts'])
def test_insert(self):
dataset1 = DataSet(data=self.description)
monitor_positions = [ (5., 5), (123,85.)]
dbgen.insert_monitor_positions(dataset1.id, monitor_positions)