Esempio n. 1
0
def extract_metadatas(images, rms_est_sigma, rms_est_fraction):
    """
    Extracts metadata and rms_qc values from the list of images.

    Args:
        images: list of image urls
        rms_est_sigma: used for RMS calculation, see `tkp.quality.statistics`
        rms_est_fraction: used for RMS calculation, see `tkp.quality.statistics`

    Returns:
        a list of metadata's. The metadata will be False if extraction failed.
    """
    results = []
    for image in images:
        logger.info("Extracting metadata from %s" % image)
        try:
            accessor = tkp.accessors.open(image)
        except TypeError as e:
            logging.error("Can't open image %s: %s" % (image, e))
            results.append(False)
        else:
            metadata = accessor.extract_metadata()
            metadata['rms_qc'] = rms_with_clipped_subregion(accessor.data,
                                                rms_est_sigma,rms_est_fraction)
            results.append(metadata)
    return results
Esempio n. 2
0
def extract_metadatas(images, rms_est_sigma, rms_est_fraction):
    """
    Extracts metadata and rms_qc values from the list of images.

    Args:
        images: list of image urls
        rms_est_sigma: used for RMS calculation, see `tkp.quality.statistics`
        rms_est_fraction: used for RMS calculation, see `tkp.quality.statistics`

    Returns:
        a list of metadata's. The metadata will be False if extraction failed.
    """
    results = []
    for image in images:
        logger.info("Extracting metadata from %s" % image)
        try:
            accessor = tkp.accessors.open(image)
        except TypeError as e:
            logging.error("Can't open image %s: %s" % (image, e))
            results.append(False)
        else:
            metadata = accessor.extract_metadata()
            metadata['rms_qc'] = rms_with_clipped_subregion(
                accessor.data, rms_est_sigma, rms_est_fraction)
            results.append(metadata)
    return results
Esempio n. 3
0
    def test_theoreticalnoise(self):
        good_image = accessors.open(good_file)
        bad_image = accessors.open(bad_file)
        frequency = good_image.freq_eff

        # this stuff should be in the header of a LOFAR image some day
        integration_time = 18654.3 # s, should be self.good_image.tau_time some day
        bandwidth = 200 * 10**3 # Hz, shoud probably be self.good_image.freq_bw some day
        ncore = 23 # ~
        nremote = 8 # ~
        nintl = 0
        configuration = "LBA_INNER"

        noise = tkp.lofar.noise.noise_level(frequency, bandwidth, integration_time, configuration, ncore, nremote, nintl)
        rms_good = statistics.rms_with_clipped_subregion(good_image.data)
        rms_bad = statistics.rms_with_clipped_subregion(bad_image.data)

        self.assertFalse(rms_invalid(rms_good, noise))
        self.assertTrue(rms_invalid(rms_bad, noise))
Esempio n. 4
0
def reject_check_lofar(accessor, job_config):

    lofar_quality_params = job_config['quality_lofar']

    low_bound = lofar_quality_params['low_bound']
    high_bound = lofar_quality_params['high_bound']
    oversampled_x = lofar_quality_params['oversampled_x']
    elliptical_x = lofar_quality_params['elliptical_x']
    min_separation = lofar_quality_params['min_separation']

    if accessor.tau_time == 0:
        logger.info("image %s REJECTED: tau_time is 0, should be > 0" %
                    accessor.url)
        return tkp.db.quality.reason['tau_time'], "tau_time is 0"

    rms_est_sigma = job_config.persistence.rms_est_sigma
    rms_est_fraction = job_config.persistence.rms_est_fraction
    rms_qc = rms_with_clipped_subregion(accessor.data,
                                        rms_est_sigma=rms_est_sigma,
                                        rms_est_fraction=rms_est_fraction)

    noise = noise_level(accessor.freq_eff, accessor.freq_bw, accessor.tau_time,
                        accessor.antenna_set, accessor.ncore, accessor.nremote,
                        accessor.nintl)
    rms_check = rms_invalid(rms_qc, noise, low_bound, high_bound)
    if not rms_check:
        logger.info("image %s accepted: rms: %s, theoretical noise: %s" % \
                        (accessor.url, nice_format(rms_qc),
                         nice_format(noise)))
    else:
        logger.info("image %s REJECTED: %s " % (accessor.url, rms_check))
        return (tkp.db.quality.reason['rms'].id, rms_check)

    # beam shape check
    (semimaj, semimin, theta) = accessor.beam
    beam_check = beam_invalid(semimaj, semimin, theta, oversampled_x,
                              elliptical_x)

    if not beam_check:
        logger.info("image %s accepted: semimaj: %s, semimin: %s" %
                    (accessor.url, nice_format(semimaj), nice_format(semimin)))
    else:
        logger.info("image %s REJECTED: %s " % (accessor.url, beam_check))
        return (tkp.db.quality.reason['beam'].id, beam_check)

    # Bright source check
    bright = tkp.quality.brightsource.is_bright_source_near(
        accessor, min_separation)
    if bright:
        logger.info("image %s REJECTED: %s " % (accessor.url, bright))
        return (tkp.db.quality.reason['bright_source'].id, bright)
Esempio n. 5
0
def reject_check_lofar(accessor, job_config):

    lofar_quality_params = job_config['quality_lofar']

    low_bound = lofar_quality_params['low_bound']
    high_bound = lofar_quality_params['high_bound']
    oversampled_x = lofar_quality_params['oversampled_x']
    elliptical_x = lofar_quality_params['elliptical_x']
    min_separation = lofar_quality_params['min_separation']

    if accessor.tau_time == 0:
        logger.info("image %s REJECTED: tau_time is 0, should be > 0" % accessor.url)
        return tkp.db.quality.reason['tau_time'], "tau_time is 0"

    rms_est_sigma = job_config.persistence.rms_est_sigma
    rms_est_fraction = job_config.persistence.rms_est_fraction
    rms_qc = rms_with_clipped_subregion(accessor.data,
                                        rms_est_sigma=rms_est_sigma,
                                        rms_est_fraction=rms_est_fraction)

    noise = noise_level(accessor.freq_eff, accessor.freq_bw, accessor.tau_time,
        accessor.antenna_set, accessor.ncore, accessor.nremote, accessor.nintl
    )
    rms_check = rms_invalid(rms_qc, noise, low_bound, high_bound)
    if not rms_check:
        logger.info("image %s accepted: rms: %s, theoretical noise: %s" % \
                        (accessor.url, nice_format(rms_qc),
                         nice_format(noise)))
    else:
        logger.info("image %s REJECTED: %s " % (accessor.url, rms_check))
        return (tkp.db.quality.reason['rms'].id, rms_check)

    # beam shape check
    (semimaj, semimin, theta) = accessor.beam
    beam_check = beam_invalid(semimaj, semimin, theta, oversampled_x, elliptical_x)

    if not beam_check:
        logger.info("image %s accepted: semimaj: %s, semimin: %s" % (accessor.url,
                                             nice_format(semimaj),
                                             nice_format(semimin)))
    else:
        logger.info("image %s REJECTED: %s " % (accessor.url, beam_check))
        return (tkp.db.quality.reason['beam'].id, beam_check)

    # Bright source check
    bright = tkp.quality.brightsource.is_bright_source_near(accessor, min_separation)
    if bright:
        logger.info("image %s REJECTED: %s " % (accessor.url, bright))
        return (tkp.db.quality.reason['bright_source'].id, bright)
Esempio n. 6
0
def reject_check_lofar(accessor, parset):
    sigma = parset['sigma']
    f = parset['f']
    low_bound = parset['low_bound']
    high_bound = parset['high_bound']
    oversampled_x = parset['oversampled_x']
    elliptical_x = parset['elliptical_x']
    min_separation = parset['min_separation']

    # RMS value check
    rms = rms_with_clipped_subregion(accessor.data, sigma, f)
    lofar_metadata = accessor.extra_metadata
    noise = noise_level(accessor.freq_eff, accessor.freq_bw, accessor.tau_time,
                    lofar_metadata['antenna_set'], lofar_metadata['ncore'],
                    lofar_metadata['nremote'], lofar_metadata['nintl'])
    rms_check = rms_invalid(rms, noise, low_bound, high_bound)
    if not rms_check:
        logger.info("image %s accepted: rms: %s, theoretical noise: %s" % \
                        (accessor.url, nice_format(rms),
                         nice_format(noise)))
    else:
        logger.info("image %s REJECTED: %s " % (accessor.url, rms_check))
        return (tkp.db.quality.reason['rms'].id, rms_check)

    # beam shape check
    (semimaj, semimin, theta) = accessor.beam
    beam_check = beam_invalid(semimaj, semimin, oversampled_x, elliptical_x)

    if not beam_check:
        logger.info("image %s accepted: semimaj: %s, semimin: %s" % (accessor.url,
                                             nice_format(semimaj),
                                             nice_format(semimin)))
    else:
        logger.info("image %s REJECTED: %s " % (accessor.url, beam_check))
        return (tkp.db.quality.reason['beam'].id, beam_check)

    # Bright source check
    bright = tkp.quality.brightsource.is_bright_source_near(accessor, min_separation)
    if bright:
        logger.info("image %s REJECTED: %s " % (accessor.url, bright))
        return (tkp.db.quality.reason['bright_source'].id, bright)
Esempio n. 7
0
 def test_rmsclippedsubregion(self):
     o = numpy.ones((800, 800))
     sub = statistics.subregion(o)
     clip = statistics.clip(sub)
     rms = statistics.rms(clip)
     self.assertEqual(rms, statistics.rms_with_clipped_subregion(o))
Esempio n. 8
0
 def test_rmsclippedsubregion(self):
     o = numpy.ones((800, 800))
     sub = statistics.subregion(o)
     clip = statistics.clip(sub)
     rms = statistics.rms(clip)
     self.assertEqual(rms, statistics.rms_with_clipped_subregion(o))