def reject_check(image_path, job_config):
""" checks if an image passes the quality check. If not, a rejection
tuple is returned.
NOTE: should only be used on a NODE
args:
id: database ID of image. This is not used but kept as a reference for
distributed computation!
image_path: path to image
parset_file: parset file location with quality check parameters
Returns:
(rejection ID, description) if rejected, else None
"""
accessor = tkp.accessors.open(image_path)
#NB could also check type, but would have to check against LofarFits,
#LofarCasa, LofarHDF...
if accessor.telescope is "LOFAR":
return reject_check_lofar(accessor,
load_section(job_config, 'quality_lofar'))
else:
logger.warn("Unrecognised telescope %s for file %s, no quality checks.",
accessor.telescope, image_path)
return None
def database_config(pipe_config, apply=False):
"""
sets up a database configuration using the settings defined in a pipeline.cfg
:param pipe_config: a ConfigParser object
:param apply: apply settings (configure db connection) or not
:return:
"""
kwargs = {}
db_parset = parset.load_section(pipe_config, 'database')
interests = ('engine', 'database', 'user', 'password', 'host', 'port',
'passphrase')
for key, value in db_parset.iteritems():
if key in interests:
kwargs[key] = value
if apply:
tkp.db.configure(**kwargs)
tkp.db.execute('select 1')
return kwargs
def run(job_name, local=False):
pipe_config = initialize_pipeline_config(
os.path.join(os.getcwd(), "pipeline.cfg"),
job_name)
database_config(pipe_config)
job_dir = pipe_config.get('layout', 'job_directory')
if not os.access(job_dir, os.X_OK):
msg = "can't access job folder %s" % job_dir
logger.error(msg)
raise IOError(msg)
logger.info("Job dir: %s", job_dir)
images = imp.load_source('images_to_process', os.path.join(job_dir,
'images_to_process.py')).images
logger.info("dataset %s contains %s images" % (job_name, len(images)))
job_config = load_job_config(pipe_config)
dump_job_config_to_logdir(pipe_config, job_config)
p_parset = parset.load_section(job_config, 'persistence')
se_parset = parset.load_section(job_config, 'source_extraction')
nd_parset = parset.load_section(job_config, 'null_detections')
tr_parset = parset.load_section(job_config, 'transient_search')
# persistence
imgs = [[img] for img in images]
arguments = [p_parset]
metadatas = runner(tasks.persistence_node_step, imgs, arguments, local)
metadatas = [m[0] for m in metadatas]
dataset_id, image_ids = steps.persistence.master_steps(metadatas,
se_parset['radius'],
p_parset)
# manual monitoringlist entries
if not add_manual_monitoringlist_entries(dataset_id, []):
return 1
images = [Image(id=image_id) for image_id in image_ids]
# quality_check
urls = [img.url for img in images]
arguments = [job_config]
rejecteds = runner(tasks.quality_reject_check, urls, arguments, local)
good_images = []
for image, rejected in zip(images, rejecteds):
if rejected:
reason, comment = rejected
steps.quality.reject_image(image.id, reason, comment)
else:
good_images.append(image)
if not good_images:
logger.warn("No good images under these quality checking criteria")
return
# Sourcefinding
urls = [img.url for img in good_images]
arguments = [se_parset]
extract_sources = runner(tasks.extract_sources, urls, arguments, local)
for image, sources in zip(good_images, extract_sources):
dbgen.insert_extracted_sources(image.id, sources, 'blind')
# null_detections
deRuiter_radius = nd_parset['deruiter_radius']
null_detectionss = [dbmon.get_nulldetections(image.id, deRuiter_radius)
for image in good_images]
iters = zip([i.url for i in good_images], null_detectionss)
arguments = [nd_parset]
ff_nds = runner(tasks.forced_fits, iters, arguments, local)
for image, ff_nd in zip(good_images, ff_nds):
dbgen.insert_extracted_sources(image.id, ff_nd, 'ff_nd')
for image in good_images:
logger.info("performing DB operations for image %s" % image.id)
dbass.associate_extracted_sources(image.id,
deRuiter_r=deRuiter_radius)
dbmon.add_nulldetections(image.id)
transients = steps.transient_search.search_transients(image.id,
tr_parset)
dbmon.adjust_transients_in_monitoringlist(image.id, transients)
for transient in transients:
steps.feature_extraction.extract_features(transient)
# ingred.classification.classify(transient, cl_parset)
now = datetime.datetime.utcnow()
dbgen.update_dataset_process_ts(dataset_id, now)
def test_all_default_parsets(self):
#This also demonstrates how we load everything into a single config
master_config = ConfigParser.SafeConfigParser()
master_config.read(default_parset_paths.values())
for section in master_config.sections():
section_params = parset.load_section(master_config, section)
