Esempio n. 1
0
def test_image_all_1000s():
    tester = ThousandsTest(None)
    image = FakeImage()
    image.data += 5
    image.data[:, :] = 1000
    image = tester.do_stage(image)
    assert image is None
Esempio n. 2
0
def test_crosstalk(set_random_seed):
    tester = CrosstalkCorrector(None)
    nx = 101
    ny = 103

    image = FakeImage(nx=nx, ny=ny)
    # Add random pixels at 10000 to each of the extensions
    for amp in range(4):
        image.data = np.ones((4, ny, nx)) * 1000.0
        random_pixels_x = np.random.randint(0, nx - 1, size=int(0.05 * nx * ny))
        random_pixels_y = np.random.randint(0, ny - 1, size=int(0.05 * nx * ny))
        for i in zip(random_pixels_y, random_pixels_x):
            image.data[amp][i] = 10000

    expected_image_data = image.data.copy()

    # Simulate crosstalk
    original_data = image.data.copy()
    for j in range(4):
        for i in range(4):
            if i != j:
                crosstalk_coeff = np.random.uniform(0.0, 0.01)
                image.header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = crosstalk_coeff
                image.data[j] += original_data[i] * crosstalk_coeff
    # Try to remove it
    image = tester.do_stage(image)
    # Assert that we got back the original image
    np.testing.assert_allclose(image.data, expected_image_data, atol=2.0, rtol=1e-5)
Esempio n. 3
0
def test_no_pixels_saturated():
    tester = SaturationTest(None)
    image = FakeImage()
    image.header['SATURATE'] = 65535
    image = tester.do_stage(image)
    assert image is not None
    assert image.header.get('SATFRAC') == 0.0
Esempio n. 4
0
def test_get_inner_quarter_default():
    test_image = FakeImage()
    test_image.data = np.random.randint(0, 1000, size=test_image.data.shape)
    # get inner quarter manually
    inner_nx = round(test_image.nx * 0.25)
    inner_ny = round(test_image.ny * 0.25)
    inner_quarter = test_image.data[inner_ny:-inner_ny, inner_nx:-inner_nx]
    np.testing.assert_array_equal(test_image.get_inner_image_section(), inner_quarter)
Esempio n. 5
0
def test_pattern_noise_on_2d_image(set_random_seed):
    image = FakeImage()
    image.data = generate_data(has_pattern_noise=True)

    detector = pattern_noise.PatternNoiseDetector(None)
    logger.error = mock.MagicMock()
    detector.do_stage(image)
    assert logger.error.called
Esempio n. 6
0
def test_pattern_noise_on_2d_image(set_random_seed):
    image = FakeImage()
    image.data = generate_data(has_pattern_noise=True)

    detector = pattern_noise.PatternNoiseDetector(None)
    logger.error = mock.MagicMock()
    detector.do_stage(image)
    assert logger.error.called
Esempio n. 7
0
def test_get_inner_quarter_default():
    test_image = FakeImage()
    test_image.data = np.random.randint(0, 1000, size=test_image.data.shape)
    # get inner quarter manually
    inner_nx = round(test_image.nx * 0.25)
    inner_ny = round(test_image.ny * 0.25)
    inner_quarter = test_image.data[inner_ny:-inner_ny, inner_nx:-inner_nx]
    np.testing.assert_array_equal(test_image.get_inner_image_section(), inner_quarter)
Esempio n. 8
0
def test_no_offset():
    tester = PointingTest(None)

    image = FakeImage()
    image.header['CRVAL1'] = '1.0'
    image.header['CRVAL2'] = '-1.0'
    image.header['OFST-RA'] = '0:04:00.00'
    image.header['OFST-DEC'] = '-01:00:00.000'

    image = tester.do_stage(image)
    np.testing.assert_allclose(image.header.get('PNTOFST'), 0.0, atol=1e-7)
Esempio n. 9
0
def test_bias_master_level_subtraction_is_reasonable(set_random_seed):
    input_bias = 2000.0
    read_noise = 15.0

    subtractor = BiasMasterLevelSubtractor(None)
    image = FakeImage()
    image.data = np.random.normal(input_bias, read_noise, size=(image.ny, image.nx))
    image = subtractor.do_stage(image)

    np.testing.assert_allclose(np.zeros(image.data.shape), image.data, atol=8 * read_noise)
    np.testing.assert_allclose(image.header.get('BIASLVL'), input_bias, atol=1.0)
Esempio n. 10
0
def test_dark_normalization_is_reasonable(set_random_seed):
    nx = 101
    ny = 103

    normalizer = DarkNormalizer(None)
    data = np.random.normal(30.0, 10, size=(ny, nx))
    image = FakeImage()
    image.data = data.copy()

    image = normalizer.do_stage(image)
    np.testing.assert_allclose(image.data, data / image.exptime, 1e-5)
Esempio n. 11
0
def test_image_30_percent_1000(set_random_seed):
    tester = ThousandsTest(None)
    nx = 101
    ny = 103
    image = FakeImage(nx=nx, ny=ny)
    random_pixels_x = np.random.randint(0, nx - 1, size=int(0.3 * nx * ny))
    random_pixels_y = np.random.randint(0, ny - 1, size=int(0.3 * nx * ny))
    for i in zip(random_pixels_y, random_pixels_x):
        image.data[i] = 1000
    image = tester.do_stage(image)
    assert image is None
Esempio n. 12
0
def test_large_offset():
    tester = PointingTest(None)

    image = FakeImage()
    image.header['CRVAL1'] = '00:00:00.000'
    image.header['CRVAL2'] = '-00:00:00.000'
    image.header['OFST-RA'] = '00:00:00.000'
    image.header['OFST-DEC'] = '-00:00:10.000'

    image = tester.do_stage(image)
    assert image.header.get('PNTOFST') == 10.0
Esempio n. 13
0
def test_pattern_noise_on_3d_image(mock_save_qc):
    data = 100.0 * np.sin(np.arange(1000000 * 4) / 0.1) + 1000.0 + np.random.normal(0.0, 10.0, size=1000000 * 4)
    data = data.reshape(4, 1000, 1000)

    image = FakeImage()
    image.data = data

    detector = pattern_noise.PatternNoiseDetector(None)
    _ = detector.do_stage([image])

    assert mock_save_qc.called
Esempio n. 14
0
def test_pattern_noise_in_only_one_quadrant(mock_save_qc):
    data = np.random.normal(0.0, 10.0, size=1000000 * 4) + 1000.0
    data = data.reshape(4, 1000, 1000)
    data[3] += 100.0 * np.sin(np.arange(1e6) / 0.1).reshape(1000, 1000)

    image = FakeImage()
    image.data = data

    detector = pattern_noise.PatternNoiseDetector(None)
    _ = detector.do_stage([image])

    assert mock_save_qc.called
Esempio n. 15
0
def test_image_10_percent_saturated_rejected(set_random_seed):
    tester = SaturationTest(None)
    nx = 101
    ny = 103
    image = FakeImage(nx=nx, ny=ny)
    image.header['SATURATE'] = 65535
    random_pixels_x = np.random.randint(0, nx - 1, size=int(0.1 * nx * ny))
    random_pixels_y = np.random.randint(0, ny - 1, size=int(0.1 * nx * ny))
    for i in zip(random_pixels_y, random_pixels_x):
        image.data[i] = image.header['SATURATE']

    image = tester.do_stage(image)
    assert image is None
Esempio n. 16
0
def test_does_not_reject_noisy_image(mock_cal, mock_frame, set_random_seed):
    mock_cal.return_value = 'test.fits'
    master_readnoise = 3.0
    nx = 101
    ny = 103

    context = make_context_with_master_bias(readnoise=master_readnoise, nx=nx, ny=ny)
    comparer = BiasComparer(context)
    image = FakeImage(image_multiplier=0.0)
    image.data = np.random.normal(0.0, image.readnoise, size=(ny, nx))

    image = comparer.do_stage(image)

    assert image.is_bad is False
Esempio n. 17
0
def test_saturation_es_update():
    # Set initial values
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    stage.save_qc_results({'Saturated': True, 'saturation_fraction': 0.99}, image)
    # Run saturation test
    tester = SaturationTest(FakeElasticsearchContext())
    image.header['SATURATE'] = 65535
    image.data += 5.0
    tester.do_stage([image])
    # Check info from elasticsearch
    results = elasticsearch.Elasticsearch(ES_URL).get_source(
        index=stage.ES_INDEX, doc_type=stage.ES_DOC_TYPE, id='test')
    assert not results['Saturated']
    assert results['saturation_fraction'] == 0.0
Esempio n. 18
0
def test_image_2_percent_saturated(set_random_seed):
    tester = SaturationTest(None)
    nx = 101
    ny = 103

    image = FakeImage(nx=nx, ny=ny)
    image.header['SATURATE'] = 65535
    random_pixels_x = np.random.randint(0, nx - 1, size=int(0.02 * nx * ny))
    random_pixels_y = np.random.randint(0, ny - 1, size=int(0.02 * nx * ny))
    for i in zip(random_pixels_y, random_pixels_x):
        image.data[i] = image.header['SATURATE']

    image = tester.do_stage(image)
    assert image is not None
    assert np.abs(image.header.get('SATFRAC') - 0.02) < 0.001
Esempio n. 19
0
def test_raises_exception_if_no_master_calibration(mock_cal, mock_images):
    mock_cal.return_value = None
    mock_images.return_value = FakeBiasImage()
    subtractor = BiasSubtractor(None)

    with pytest.raises(MasterCalibrationDoesNotExist):
        images = subtractor.do_stage([FakeImage() for x in range(6)])
Esempio n. 20
0
def test_save_results():
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    output = stage.save_qc_results({}, image, _source=True)
    assert '_shards' in output
    assert output['_shards']['failed'] == 0
    assert 'get' in output
Esempio n. 21
0
def test_save_qc_results(mock_es):
    context = FakeContext()
    context.post_to_elasticsearch = True
    context.elasticsearch_url = '/'
    stage = FakeStage(context)
    qc.save_qc_results(stage.runtime_context, {}, FakeImage())
    assert mock_es.called
Esempio n. 22
0
def test_crosstalk():
    tester = CrosstalkCorrector(None)
    nx = 101
    ny = 103

    images = [FakeImage(nx=nx, ny=ny) for x in range(6)]
    # Add random pixels at 10000 to each of the extensions
    for image in images:
        for amp in range(4):
            image.data = np.ones((4, ny, nx)) * 1000.0
            random_pixels_x = np.random.randint(0, nx - 1, size=int(0.05 * nx * ny))
            random_pixels_y = np.random.randint(0, ny - 1, size=int(0.05 * nx * ny))
            for i in zip(random_pixels_y, random_pixels_x):
                image.data[amp][i] = 10000

    expected_image_data = [image.data.copy() for image in images]

    # Simulate crosstalk
    for image in images:
        original_data = image.data.copy()
        for j in range(4):
            for i in range(4):
                if i != j:
                    crosstalk_coeff = np.random.uniform(0.0, 0.01)
                    image.header['CRSTLK{i}{j}'.format(i=i+1, j=j+1)] = crosstalk_coeff
                    image.data[j] += original_data[i] * crosstalk_coeff
    # Try to remove it
    images = tester.do_stage(images)
    # Assert that we got back the original image
    for i, image in enumerate(images):
        np.testing.assert_allclose(image.data, expected_image_data[i], atol=2.0, rtol=1e-5)
Esempio n. 23
0
def test_does_not_reject_noisy_image(mock_cal, mock_frame, set_random_seed):
    mock_cal.return_value = 'test.fits'
    master_readnoise = 3.0
    nx = 101
    ny = 103

    context = make_context_with_master_bias(readnoise=master_readnoise,
                                            nx=nx,
                                            ny=ny)
    comparer = BiasComparer(context)
    image = FakeImage(image_multiplier=0.0)
    image.data = np.random.normal(0.0, image.readnoise, size=(ny, nx))

    image = comparer.do_stage(image)

    assert image.is_bad is False
Esempio n. 24
0
def test_format_qc_results_new_info():
    filename, results = qc.format_qc_results(
        {
            "key1": "value1",
            "key2": "value2"
        }, FakeImage())
    assert results["key1"] == "value1"
    assert results["key2"] == "value2"
Esempio n. 25
0
def test_format_qc_results_numpy_bool():
    filename, results = qc.format_qc_results(
        {
            "normal_bool": True,
            "numpy_bool": np.bool_(True)
        }, FakeImage())
    assert type(results["normal_bool"]) == bool
    assert type(results["numpy_bool"]) == bool
Esempio n. 26
0
def test_flat_normalization_is_reasonable(set_random_seed):
    flat_variation = 0.05
    input_level = 10000.0
    nx = 101
    ny = 103

    normalizer = FlatNormalizer(None)
    image = FakeImage()
    flat_pattern = np.random.normal(1.0, flat_variation, size=(ny, nx))
    image.data = np.random.poisson(flat_pattern * input_level).astype(float)
    image = normalizer.do_stage(image)

    # For right now, we only use a quarter of the image to calculate the flat normalization
    # because real ccds have crazy stuff at the edges, so the S/N is cut down by a factor of 2
    # Assume 50% slop because the variation in the pattern does not decrease like sqrt(n)
    assert np.abs(image.header['FLATLVL'] - input_level) < (3.0 * flat_variation * input_level / (nx * ny) ** 0.5)
    assert np.abs(np.mean(image.data) - 1.0) <= 3.0 * flat_variation / (nx * ny) ** 0.5
Esempio n. 27
0
def test_save_results_with_changed_parameters():
    # Push the defualt test image info
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    stage.save_qc_results({}, image)
    # Change the image parameters and push again
    image.site = "fake_site"
    image.instrument = "fake_instrument"
    image.epoch = str(int(image.epoch) + 1)
    image.dateobs += timedelta(days=1)
    # Post to ES and test results
    output = stage.save_qc_results({}, image, _source=True)
    assert output['result'] == 'updated'
    results = output['get']['_source']
    assert results['site'] == image.site
    assert results['instrument'] == image.instrument
    assert results['dayobs'] == image.epoch
    assert results['timestamp'] == image.dateobs.strftime("%Y-%m-%dT%H:%M:%S")
Esempio n. 28
0
def test_does_flag_bad_image(mock_cal, mock_frame, set_random_seed):
    mock_cal.return_value = 'test.fits'
    master_readnoise = 3.0
    nx = 101
    ny = 103

    context = make_context_with_master_bias(readnoise=master_readnoise, nx=nx, ny=ny)
    comparer = BiasComparer(context)
    image = FakeImage(image_multiplier=0.0)
    image.data = np.random.normal(0.0, image.readnoise, size=(ny, nx))

    x_indexes = np.random.choice(np.arange(nx), size=2000)
    y_indexes = np.random.choice(np.arange(ny), size=2000)
    for x, y in zip(x_indexes, y_indexes):
        image.data[y, x] = np.random.normal(100, image.readnoise)
    image = comparer.do_stage(image)

    assert image.is_bad
Esempio n. 29
0
def test_flags_bad_if_no_master_calibration(mock_cal, mock_frame,
                                            set_random_seed):
    mock_cal.return_value = None
    nx = 101
    ny = 103
    context = make_context_with_master_bias(nx=nx, ny=ny)
    comparer = BiasComparer(context)
    image = comparer.do_stage(FakeImage(nx=nx, ny=ny))
    assert image.is_bad is True
Esempio n. 30
0
def test_format_qc_results_basic_info():
    image = FakeImage()
    filename, results = qc.format_qc_results({}, image)
    assert results['site'] == image.site
    assert results['instrument'] == image.camera
    assert results['dayobs'] == image.epoch
    assert results['@timestamp'] == image.dateobs
    assert results['obstype'] == image.obstype
    assert filename in image.filename
Esempio n. 31
0
def test_no_pixels_1000():
    tester = ThousandsTest(None)
    nx = 101
    ny = 103

    images = [FakeImage(nx=nx, ny=ny) for x in range(6)]

    images = tester.do_stage(images)

    assert len(images) == 6
Esempio n. 32
0
def test_does_flag_bad_image(mock_cal, mock_frame, set_random_seed):
    mock_cal.return_value = 'test.fits'
    master_readnoise = 3.0
    nx = 101
    ny = 103

    context = make_context_with_master_bias(readnoise=master_readnoise,
                                            nx=nx,
                                            ny=ny)
    comparer = BiasComparer(context)
    image = FakeImage(image_multiplier=0.0)
    image.data = np.random.normal(0.0, image.readnoise, size=(ny, nx))

    x_indexes = np.random.choice(np.arange(nx), size=2000)
    y_indexes = np.random.choice(np.arange(ny), size=2000)
    for x, y in zip(x_indexes, y_indexes):
        image.data[y, x] = np.random.normal(100, image.readnoise)
    image = comparer.do_stage(image)

    assert image.is_bad
Esempio n. 33
0
def test_header_checker_es_update():
    # Set initial values
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    header_checks = ['HeaderBadDecValue', 'HeaderBadRAValue',
                     'HeaderExptimeNegative', 'HeaderExptimeZero',
                     'HeaderKeywordsMissing', 'HeaderKeywordsNA']
    header_check_booleans = {key: True for key in header_checks}
    stage.save_qc_results(header_check_booleans, image)
    # Run header sanity test
    tester = HeaderSanity(FakeElasticsearchContext())
    for key in tester.header_expected_format.keys():
        image.header[key] = 1.0
    image.header['OBSTYPE'] = 'EXPOSE'
    tester.do_stage([image])
    # Check info from elasticsearch
    results = elasticsearch.Elasticsearch(ES_URL).get_source(
        index=stage.ES_INDEX, doc_type=stage.ES_DOC_TYPE, id='test')
    for header_check in header_checks:
        assert not results[header_check]
Esempio n. 34
0
def test_pointing_es_update():
    # Set initial values
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    stage.save_qc_results({'PointingSevere': True,
                              'PointingWarning': True,
                              'pointing_offset': 100.}, image)
    # Run pointing test
    image.header['CRVAL1'] = '1.0'
    image.header['CRVAL2'] = '-1.0'
    image.header['OFST-RA'] = '0:04:00.00'
    image.header['OFST-DEC'] = '-01:00:00.000'
    tester = PointingTest(FakeElasticsearchContext())
    tester.do_stage([image])
    # Check info from elasticsearch
    results = elasticsearch.Elasticsearch(ES_URL).get_source(
        index=stage.ES_INDEX, doc_type=stage.ES_DOC_TYPE, id='test')
    assert not results['PointingSevere']
    assert not results['PointingWarning']
    assert results['pointing_offset'] < 1E10
Esempio n. 35
0
def test_nonzero_but_no_pixels_1000():
    tester = ThousandsTest(None)
    nx = 101
    ny = 103

    images = [FakeImage(nx=nx, ny=ny) for x in range(6)]
    for image in images:
        image.data += 5

    images = tester.do_stage(images)

    assert len(images) == 6
Esempio n. 36
0
def test_pattern_noise_es_update():
    # Set initial values
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    stage.save_qc_results({'PatternNoise': True}, image)
    # Run pattern noise test
    tester = PatternNoiseDetector(FakeElasticsearchContext())
    tester.do_stage([image])
    # Check info from elasticsearch
    results = elasticsearch.Elasticsearch(ES_URL).get_source(
        index=stage.ES_INDEX, doc_type=stage.ES_DOC_TYPE, id='test')
    assert not results['PatternNoise']
Esempio n. 37
0
def test_all_images_all_1000s():
    tester = ThousandsTest(None)
    nx = 101
    ny = 103

    images = [FakeImage(nx=nx, ny=ny) for x in range(6)]
    for image in images:
        image.data[:, :] = 1000

    images = tester.do_stage(images)

    assert len(images) == 0
Esempio n. 38
0
def test_sinistro_1000s_es_update():
    # Set initial values
    stage = Stage(FakeElasticsearchContext())
    image = FakeImage()
    stage.save_qc_results({'Error1000s': True, 'fraction_1000s': 0.99}, image)
    # Run sinistro 1000s test
    tester = ThousandsTest(FakeElasticsearchContext())
    tester.do_stage([image])
    # Check info from elasticsearch
    results = elasticsearch.Elasticsearch(ES_URL).get_source(
        index=stage.ES_INDEX, doc_type=stage.ES_DOC_TYPE, id='test')
    assert not results['Error1000s']
    assert results['fraction_1000s'] == 0.0
Esempio n. 39
0
def test_no_pixels_saturated():
    tester = SaturationTest(None)
    nx = 101
    ny = 103

    images = [FakeImage(nx=nx, ny=ny) for x in range(6)]
    for image in images:
        image.header['SATURATE'] = 65535

    images = tester.do_stage(images)
    for image in images:
        assert image.header['SATFRAC'][0] == 0.0
    assert len(images) == 6
Esempio n. 40
0
def test_2d_is_not_3d():
    test_image = FakeImage()
    assert not test_image.data_is_3d()
Esempio n. 41
0
def test_3d_is_3d():
    test_image = FakeImage(n_amps=4)
    assert test_image.data_is_3d()
Esempio n. 42
0
def test_get_n_amps_2d():
    n_amps = 4
    test_image = FakeImage(n_amps=n_amps)
    assert test_image.get_n_amps() == n_amps
Esempio n. 43
0
def test_get_n_amps_3d():
    test_image = FakeImage()
    assert test_image.get_n_amps() == 1
Esempio n. 44
0
def test_get_inner_image_section_3d():
    test_image = FakeImage(n_amps=4)
    with pytest.raises(ValueError):
        test_image.get_inner_image_section()