def extract_metadatas(images, rms_est_sigma, rms_est_fraction):
"""
Extracts metadata and rms_qc values from the list of images.
Args:
images: list of image urls
rms_est_sigma: used for RMS calculation, see `tkp.quality.statistics`
rms_est_fraction: used for RMS calculation, see `tkp.quality.statistics`
Returns:
a list of metadata's. The metadata will be False if extraction failed.
"""
results = []
for image in images:
logger.info("Extracting metadata from %s" % image)
try:
accessor = tkp.accessors.open(image)
except TypeError as e:
logging.error("Can't open image %s: %s" % (image, e))
results.append(False)
else:
metadata = accessor.extract_metadata()
metadata['rms_qc'] = rms_with_clipped_subregion(accessor.data,
rms_est_sigma,rms_est_fraction)
results.append(metadata)
return results
def extract_metadatas(images, rms_est_sigma, rms_est_fraction):
"""
Extracts metadata and rms_qc values from the list of images.
Args:
images: list of image urls
rms_est_sigma: used for RMS calculation, see `tkp.quality.statistics`
rms_est_fraction: used for RMS calculation, see `tkp.quality.statistics`
Returns:
a list of metadata's. The metadata will be False if extraction failed.
"""
results = []
for image in images:
logger.info("Extracting metadata from %s" % image)
try:
accessor = tkp.accessors.open(image)
except TypeError as e:
logging.error("Can't open image %s: %s" % (image, e))
results.append(False)
else:
metadata = accessor.extract_metadata()
metadata['rms_qc'] = rms_with_clipped_subregion(
accessor.data, rms_est_sigma, rms_est_fraction)
results.append(metadata)
return results
def test_theoreticalnoise(self):
good_image = accessors.open(good_file)
bad_image = accessors.open(bad_file)
frequency = good_image.freq_eff
# this stuff should be in the header of a LOFAR image some day
integration_time = 18654.3 # s, should be self.good_image.tau_time some day
bandwidth = 200 * 10**3 # Hz, shoud probably be self.good_image.freq_bw some day
ncore = 23 # ~
nremote = 8 # ~
nintl = 0
configuration = "LBA_INNER"
noise = tkp.lofar.noise.noise_level(frequency, bandwidth, integration_time, configuration, ncore, nremote, nintl)
rms_good = statistics.rms_with_clipped_subregion(good_image.data)
rms_bad = statistics.rms_with_clipped_subregion(bad_image.data)
self.assertFalse(rms_invalid(rms_good, noise))
self.assertTrue(rms_invalid(rms_bad, noise))
def reject_check_lofar(accessor, job_config):
lofar_quality_params = job_config['quality_lofar']
low_bound = lofar_quality_params['low_bound']
high_bound = lofar_quality_params['high_bound']
oversampled_x = lofar_quality_params['oversampled_x']
elliptical_x = lofar_quality_params['elliptical_x']
min_separation = lofar_quality_params['min_separation']
if accessor.tau_time == 0:
logger.info("image %s REJECTED: tau_time is 0, should be > 0" %
accessor.url)
return tkp.db.quality.reason['tau_time'], "tau_time is 0"
rms_est_sigma = job_config.persistence.rms_est_sigma
rms_est_fraction = job_config.persistence.rms_est_fraction
rms_qc = rms_with_clipped_subregion(accessor.data,
rms_est_sigma=rms_est_sigma,
rms_est_fraction=rms_est_fraction)
noise = noise_level(accessor.freq_eff, accessor.freq_bw, accessor.tau_time,
accessor.antenna_set, accessor.ncore, accessor.nremote,
accessor.nintl)
rms_check = rms_invalid(rms_qc, noise, low_bound, high_bound)
if not rms_check:
logger.info("image %s accepted: rms: %s, theoretical noise: %s" % \
(accessor.url, nice_format(rms_qc),
nice_format(noise)))
else:
logger.info("image %s REJECTED: %s " % (accessor.url, rms_check))
return (tkp.db.quality.reason['rms'].id, rms_check)
# beam shape check
(semimaj, semimin, theta) = accessor.beam
beam_check = beam_invalid(semimaj, semimin, theta, oversampled_x,
elliptical_x)
if not beam_check:
logger.info("image %s accepted: semimaj: %s, semimin: %s" %
(accessor.url, nice_format(semimaj), nice_format(semimin)))
else:
logger.info("image %s REJECTED: %s " % (accessor.url, beam_check))
return (tkp.db.quality.reason['beam'].id, beam_check)
# Bright source check
bright = tkp.quality.brightsource.is_bright_source_near(
accessor, min_separation)
if bright:
logger.info("image %s REJECTED: %s " % (accessor.url, bright))
return (tkp.db.quality.reason['bright_source'].id, bright)
def reject_check_lofar(accessor, job_config):
lofar_quality_params = job_config['quality_lofar']
low_bound = lofar_quality_params['low_bound']
high_bound = lofar_quality_params['high_bound']
oversampled_x = lofar_quality_params['oversampled_x']
elliptical_x = lofar_quality_params['elliptical_x']
min_separation = lofar_quality_params['min_separation']
if accessor.tau_time == 0:
logger.info("image %s REJECTED: tau_time is 0, should be > 0" % accessor.url)
return tkp.db.quality.reason['tau_time'], "tau_time is 0"
rms_est_sigma = job_config.persistence.rms_est_sigma
rms_est_fraction = job_config.persistence.rms_est_fraction
rms_qc = rms_with_clipped_subregion(accessor.data,
rms_est_sigma=rms_est_sigma,
rms_est_fraction=rms_est_fraction)
noise = noise_level(accessor.freq_eff, accessor.freq_bw, accessor.tau_time,
accessor.antenna_set, accessor.ncore, accessor.nremote, accessor.nintl
)
rms_check = rms_invalid(rms_qc, noise, low_bound, high_bound)
if not rms_check:
logger.info("image %s accepted: rms: %s, theoretical noise: %s" % \
(accessor.url, nice_format(rms_qc),
nice_format(noise)))
else:
logger.info("image %s REJECTED: %s " % (accessor.url, rms_check))
return (tkp.db.quality.reason['rms'].id, rms_check)
# beam shape check
(semimaj, semimin, theta) = accessor.beam
beam_check = beam_invalid(semimaj, semimin, theta, oversampled_x, elliptical_x)
if not beam_check:
logger.info("image %s accepted: semimaj: %s, semimin: %s" % (accessor.url,
nice_format(semimaj),
nice_format(semimin)))
else:
logger.info("image %s REJECTED: %s " % (accessor.url, beam_check))
return (tkp.db.quality.reason['beam'].id, beam_check)
# Bright source check
bright = tkp.quality.brightsource.is_bright_source_near(accessor, min_separation)
if bright:
logger.info("image %s REJECTED: %s " % (accessor.url, bright))
return (tkp.db.quality.reason['bright_source'].id, bright)
def reject_check_lofar(accessor, parset):
sigma = parset['sigma']
f = parset['f']
low_bound = parset['low_bound']
high_bound = parset['high_bound']
oversampled_x = parset['oversampled_x']
elliptical_x = parset['elliptical_x']
min_separation = parset['min_separation']
# RMS value check
rms = rms_with_clipped_subregion(accessor.data, sigma, f)
lofar_metadata = accessor.extra_metadata
noise = noise_level(accessor.freq_eff, accessor.freq_bw, accessor.tau_time,
lofar_metadata['antenna_set'], lofar_metadata['ncore'],
lofar_metadata['nremote'], lofar_metadata['nintl'])
rms_check = rms_invalid(rms, noise, low_bound, high_bound)
if not rms_check:
logger.info("image %s accepted: rms: %s, theoretical noise: %s" % \
(accessor.url, nice_format(rms),
nice_format(noise)))
else:
logger.info("image %s REJECTED: %s " % (accessor.url, rms_check))
return (tkp.db.quality.reason['rms'].id, rms_check)
# beam shape check
(semimaj, semimin, theta) = accessor.beam
beam_check = beam_invalid(semimaj, semimin, oversampled_x, elliptical_x)
if not beam_check:
logger.info("image %s accepted: semimaj: %s, semimin: %s" % (accessor.url,
nice_format(semimaj),
nice_format(semimin)))
else:
logger.info("image %s REJECTED: %s " % (accessor.url, beam_check))
return (tkp.db.quality.reason['beam'].id, beam_check)
# Bright source check
bright = tkp.quality.brightsource.is_bright_source_near(accessor, min_separation)
if bright:
logger.info("image %s REJECTED: %s " % (accessor.url, bright))
return (tkp.db.quality.reason['bright_source'].id, bright)
def test_rmsclippedsubregion(self):
o = numpy.ones((800, 800))
sub = statistics.subregion(o)
clip = statistics.clip(sub)
rms = statistics.rms(clip)
self.assertEqual(rms, statistics.rms_with_clipped_subregion(o))
def test_rmsclippedsubregion(self):
o = numpy.ones((800, 800))
sub = statistics.subregion(o)
clip = statistics.clip(sub)
rms = statistics.rms(clip)
self.assertEqual(rms, statistics.rms_with_clipped_subregion(o))