def test_headers_with_filter_missing_keyword(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp'])
for header in collection.headers(imagetyp='light', object=''):
assert header['imagetyp'].lower() == 'light'
with pytest.raises(KeyError):
header['object']
def test_headers_with_filter_wildcard(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp'])
cnt = 0
for header in collection.headers(imagetyp='*'):
cnt += 1
assert cnt == triage_setup.n_test['files']
def test_files_with_no_compressed(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir)
n_files_found = len(
collection._fits_files_in_directory(compressed=False))
n_uncompressed = (triage_setup.n_test['files'] -
triage_setup.n_test['compressed'])
assert n_files_found == n_uncompressed
def test_type_of_empty_collection(self, triage_setup):
# Test for implementation of the suggestion in
#
# https://github.com/astropy/ccdproc/issues/601
#
# in which an empty collection with no keys has but with files
# returns a summary table with one column, but an empty collection
# with no keys and no files returns None.
# Make a dummy keyword that we then delete.
with catch_warnings(AstropyUserWarning) as warning_lines:
ic = ImageFileCollection(triage_setup.test_dir, keywords=['fafa'])
assert "no FITS files in the collection."
ic.keywords = []
assert set(ic.summary.colnames) == set(['file'])
# Remove all of the fits files
path = Path(triage_setup.test_dir)
for p in path.iterdir():
p.unlink()
# Now the summary should be none
ic = ImageFileCollection(triage_setup.test_dir)
assert ic.summary is None
assert ic.keywords == []
def test_hdus_masking(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp', 'exposure'])
old_data = np.array(collection.summary)
for hdu in collection.hdus(imagetyp='bias'):
pass
new_data = np.array(collection.summary)
assert (new_data == old_data).all()
def test_headers_with_filter(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp'])
cnt = 0
for header in collection.headers(imagetyp='light'):
assert header['imagetyp'].lower() == 'light'
cnt += 1
assert cnt == triage_setup.n_test['light']
def test_consecutive_fiilters(self, triage_setup):
collection = ImageFileCollection(
location=triage_setup.test_dir,
keywords=['imagetyp', 'filter', 'object'])
no_files_match = collection.files_filtered(object='fdsafs')
assert (len(no_files_match) == 0)
some_files_should_match = collection.files_filtered(object=None,
imagetyp='light')
assert (len(some_files_should_match) == triage_setup.n_test['light'])
def test_initialization_with_no_keywords(self, triage_setup):
# This test is primarily historical -- the old default for
# keywords was an empty list (it is now the wildcard '*').
ic = ImageFileCollection(location=triage_setup.test_dir, keywords=[])
# iteration below failed before bugfix...
execs = 0
for h in ic.headers():
execs += 1
assert not execs
def test_filter_files(self, triage_setup):
img_collection = ImageFileCollection(
location=triage_setup.test_dir, keywords=['imagetyp', 'filter'])
assert len(img_collection.files_filtered(
imagetyp='bias')) == triage_setup.n_test['bias']
assert len(img_collection.files) == triage_setup.n_test['files']
assert ('filter' in img_collection.keywords)
assert ('flying monkeys' not in img_collection.keywords)
assert len(img_collection.values('imagetyp', unique=True)) == 2
def test_headers(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp'])
n_headers = 0
for header in collection.headers():
assert isinstance(header, fits.Header)
assert ('bzero' in header)
n_headers += 1
assert n_headers == triage_setup.n_test['files']
def test_setting_write_location_to_bad_dest_raises_error(self, tmpdir,
triage_setup):
new_tmp = tmpdir.mkdtemp()
bad_directory = new_tmp.join('foo')
ic = ImageFileCollection(triage_setup.test_dir, keywords=['naxis'])
with pytest.raises(IOError):
for hdr in ic.headers(save_location=bad_directory.strpath):
pass
def test_summary_table_is_always_masked(self, triage_setup):
# First, try grabbing all of the keywords
ic = ImageFileCollection(location=triage_setup.test_dir, keywords='*')
assert ic.summary.masked
# Now, try keywords that every file will have
ic.keywords = ['bitpix']
assert ic.summary.masked
# What about keywords that include some that will surely be missing?
ic.keywords = ['bitpix', 'dsafui']
assert ic.summary.masked
def test_filter_does_not_not_permanently_change_file_mask(self,
triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp'])
# ensure all files are originally unmasked
assert not collection.summary['file'].mask.any()
# generate list that will match NO files
collection.files_filtered(imagetyp='foisajfoisaj')
# if the code works, this should have no permanent effect
assert not collection.summary['file'].mask.any()
def test_headers_with_multiple_filters(self, triage_setup):
collection = ImageFileCollection(location=triage_setup.test_dir,
keywords=['imagetyp'])
cnt = 0
for header in collection.headers(imagetyp='light', filter='R'):
assert header['imagetyp'].lower() == 'light'
assert header['filter'].lower() == 'r'
cnt += 1
assert cnt == (triage_setup.n_test['light'] -
triage_setup.n_test['missing_filter_value'])
def test_generator_with_regex(self, triage_setup):
ic = ImageFileCollection(triage_setup.test_dir)
n_light = 0
for h in ic.headers(regex_match=True, imagetyp='li.*t'):
assert h['imagetyp'].lower() == 'light'
n_light += 1
assert n_light == triage_setup.n_test['light']
def test_refresh_method_sees_added_files(self, triage_setup):
ic = ImageFileCollection(triage_setup.test_dir, keywords='*')
# Compressed files don't get copied. Not sure why...
original_len = len(ic.summary) - triage_setup.n_test['compressed']
# Generate additional files in this directory
for h in ic.headers(save_with_name="_foo"):
pass
ic.refresh()
new_len = len(ic.summary) - triage_setup.n_test['compressed']
assert new_len == 2 * original_len
def test_regex_match_for_search(self, triage_setup):
# Test regex matching in searches
ic = ImageFileCollection(triage_setup.test_dir)
files = ic.files_filtered(regex_match=True, imagetyp='b.*s')
assert len(files) == triage_setup.n_test['bias']
# This should return all of the files in the test set
all_files = ic.files_filtered(regex_match=True, imagetyp='bias|light')
assert len(all_files) == triage_setup.n_test['files']
# Add a column with more interesting content and see whether we
# match that.
ic.summary['match_me'] = [
'hello', 'goodbye', 'bye', 'byte', 'good bye hello', 'dog'
]
hello_anywhere = ic.files_filtered(regex_match=True, match_me='hello')
assert len(hello_anywhere) == 2
hello_start = ic.files_filtered(regex_match=True, match_me='^hello')
assert len(hello_start) == 1
# Is it really a case-insensitive match?
hello_start = ic.files_filtered(regex_match=True, match_me='^HeLlo')
assert len(hello_start) == 1
any_bye = ic.files_filtered(regex_match=True, match_me='by.*e')
assert len(any_bye) == 4
def test_filtered_files_have_proper_path(self, triage_setup):
ic = ImageFileCollection(location=triage_setup.test_dir, keywords='*')
# Get a subset of the files.
plain_biases = ic.files_filtered(imagetyp='bias')
# Force a copy...
plain_biases = list(plain_biases)
# Same subset, but with full path.
path_biases = ic.files_filtered(imagetyp='bias', include_path=True)
for path_b, plain_b in zip(path_biases, plain_biases):
# If the path munging has been done properly, this will succeed.
assert os.path.basename(path_b) == plain_b
def test_fits_summary_when_keywords_are_not_subset(self, triage_setup):
"""
Catch case when there is overlap between keyword list
passed to the ImageFileCollection and to files_filtered
but the latter is not a subset of the former.
"""
ic = ImageFileCollection(triage_setup.test_dir,
keywords=['imagetyp', 'exposure'])
n_files = len(ic.files)
files_missing_this_key = ic.files_filtered(imagetyp='*', monkeys=None)
assert (n_files > 0)
assert (n_files == len(files_missing_this_key))
def test_dir_with_no_fits_files(self, tmpdir):
empty_dir = tmpdir.mkdtemp()
some_file = empty_dir.join('some_file.txt')
some_file.dump('words')
with catch_warnings() as w:
collection = ImageFileCollection(location=empty_dir.strpath,
keywords=['imagetyp'])
assert len(w) == 1
assert str(w[0].message) == "no FITS files in the collection."
assert collection.summary is None
for hdr in collection.headers():
# this statement should not be reached if there are no FITS files
assert 0
def test_fits_summary(triage_setup):
keywords = ['imagetyp', 'filter']
ic = ImageFileCollection(triage_setup.test_dir, keywords=keywords)
summary = ic._fits_summary(header_keywords=keywords)
assert len(summary['file']) == triage_setup.n_test['files']
for keyword in keywords:
assert len(summary[keyword]) == triage_setup.n_test['files']
# explicit conversion to array is needed to avoid astropy Table bug in
# 0.2.4
no_filter_no_object_row = np.array(
summary['file'] == 'no_filter_no_object_bias.fit')
# there should be no filter keyword in the bias file
assert summary['filter'][no_filter_no_object_row].mask
def test_make_collection_by_filtering(self, triage_setup):
# Test for implementation of feature at
#
# https://github.com/astropy/ccdproc/issues/596
#
# which adds the ability to create a new collection by filtering
# an existing ImageFileCollection.
ic = ImageFileCollection(location=triage_setup.test_dir)
new_ic = ic.filter(imagetyp='light')
assert len(new_ic.summary) == triage_setup.n_test['light']
for header in new_ic.headers():
assert header['imagetyp'].lower() == 'light'
def test_refresh_method_sees_added_keywords(self, triage_setup):
ic = ImageFileCollection(triage_setup.test_dir, keywords='*')
# Add a keyword I know isn't already in the header to each file.
not_in_header = 'BARKARK'
for h in ic.headers(overwrite=True):
h[not_in_header] = True
assert not_in_header not in ic.summary.colnames
ic.refresh()
# After refreshing the odd keyword should be present.
assert not_in_header.lower() in ic.summary.colnames
def test_filter_files_whitespace_keys(self, triage_setup):
hdr = fits.Header([('HIERARCH a b', 2)])
hdul = fits.HDUList([fits.PrimaryHDU(np.ones((10, 10)), header=hdr)])
hdul.writeto(
os.path.join(triage_setup.test_dir, 'hdr_with_whitespace.fits'))
ic = ImageFileCollection(location=triage_setup.test_dir)
# Using a dictionary and unpacking it should work
filtered = ic.files_filtered(**{'a b': 2})
assert len(filtered) == 1
assert 'hdr_with_whitespace.fits' in filtered
# Also check it's working with generators:
for _, filename in ic.data(a_b=2, replace_='_', return_fname=True):
assert filename == 'hdr_with_whitespace.fits'
def test_hdus(self, triage_setup):
collection = ImageFileCollection(
location=triage_setup.test_dir, keywords=['imagetyp'])
n_hdus = 0
for hdu in collection.hdus():
assert isinstance(hdu, fits.PrimaryHDU)
data = hdu.data # must access the data to force scaling
# pre-astropy 1.1 unsigned data was changed to float32 and BZERO
# removed. In 1.1 and later, BZERO stays but the data type is
# unsigned int.
assert (('BZERO' not in hdu.header) or
(data.dtype is np.dtype(np.uint16)))
n_hdus += 1
assert n_hdus == triage_setup.n_test['files']
def test_force_detect_fits_files_finds_fits_files(self, triage_setup):
# Tests for new feature
#
# https://github.com/astropy/ccdproc/issues/620
#
# which supports adding all of the FITS files in a location based on
# their *contents* instead of their *extension*.
# Grab files from the default collection and make a copy with a new name
# (and with no fits-like extension)
#
# Making a copy of *every* file means we can just double the expected
# number of files as part of the tests.
path = Path(triage_setup.test_dir)
for idx, p in enumerate(path.iterdir()):
new_name = 'no_extension{}'.format(idx)
new_path = path / new_name
new_path.write_bytes(p.read_bytes())
ic = ImageFileCollection(location=str(path),
find_fits_by_reading=True)
# Compressed files won't be automagically detected by reading the
# first few bytes.
expected_number = (2 * triage_setup.n_test['files'] -
triage_setup.n_test['compressed'])
assert len(ic.summary) == expected_number
n_bias = (ic.summary['imagetyp'] == 'BIAS').sum()
assert n_bias == 2 * triage_setup.n_test['bias']
# Only one file in the original set of test files has exposure time
# 15, so there should be two now.
assert len(ic.files_filtered(exposure=15.0)) == 2
# Try one of the generators
expected_heads = (2 * triage_setup.n_test['light'] -
triage_setup.n_test['compressed'])
n_heads = 0
for h in ic.headers(imagetyp='light'):
assert h['imagetyp'].lower() == 'light'
n_heads += 1
assert n_heads == expected_heads
def test_duplicate_keywords_in_setting(self, triage_setup):
keywords_in = ['imagetyp', 'a', 'a']
ic = ImageFileCollection(triage_setup.test_dir, keywords=keywords_in)
for key in set(keywords_in):
assert (key in ic.keywords)
# one keyword gets added: file
assert len(ic.keywords) < len(keywords_in) + 1
def test_keyword_includes_file(self, triage_setup):
keywords_in = ['file', 'imagetyp']
ic = ImageFileCollection(triage_setup.test_dir,
keywords=keywords_in)
assert 'file' in ic.keywords
file_keywords = [key for key in ic.keywords if key == 'file']
assert len(file_keywords) == 1
def test_repr_globs(self, triage_setup):
ic = ImageFileCollection(location=triage_setup.test_dir,
glob_exclude="*no_filter*",
glob_include="*object_light*")
ref = ("ImageFileCollection(location={0!r}, "
"glob_include='*object_light*', "
"glob_exclude='*no_filter*')".format(triage_setup.test_dir))
assert repr(ic) == ref
def test_collection_when_one_file_not_fits(self, triage_setup):
not_fits = 'foo.fit'
path_bad = os.path.join(triage_setup.test_dir, not_fits)
# create an empty file...
with open(path_bad, 'w'):
pass
ic = ImageFileCollection(triage_setup.test_dir, keywords=['imagetyp'])
assert not_fits not in ic.summary['file']
os.remove(path_bad)
