def test_headers_with_filter_missing_keyword(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp']) for header in collection.headers(imagetyp='light', object=''): assert header['imagetyp'].lower() == 'light' with pytest.raises(KeyError): header['object']
def test_headers_with_filter_wildcard(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp']) cnt = 0 for header in collection.headers(imagetyp='*'): cnt += 1 assert cnt == triage_setup.n_test['files']
def test_files_with_no_compressed(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir) n_files_found = len( collection._fits_files_in_directory(compressed=False)) n_uncompressed = (triage_setup.n_test['files'] - triage_setup.n_test['compressed']) assert n_files_found == n_uncompressed
def test_type_of_empty_collection(self, triage_setup): # Test for implementation of the suggestion in # # https://github.com/astropy/ccdproc/issues/601 # # in which an empty collection with no keys has but with files # returns a summary table with one column, but an empty collection # with no keys and no files returns None. # Make a dummy keyword that we then delete. with catch_warnings(AstropyUserWarning) as warning_lines: ic = ImageFileCollection(triage_setup.test_dir, keywords=['fafa']) assert "no FITS files in the collection." ic.keywords = [] assert set(ic.summary.colnames) == set(['file']) # Remove all of the fits files path = Path(triage_setup.test_dir) for p in path.iterdir(): p.unlink() # Now the summary should be none ic = ImageFileCollection(triage_setup.test_dir) assert ic.summary is None assert ic.keywords == []
def test_hdus_masking(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp', 'exposure']) old_data = np.array(collection.summary) for hdu in collection.hdus(imagetyp='bias'): pass new_data = np.array(collection.summary) assert (new_data == old_data).all()
def test_headers_with_filter(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp']) cnt = 0 for header in collection.headers(imagetyp='light'): assert header['imagetyp'].lower() == 'light' cnt += 1 assert cnt == triage_setup.n_test['light']
def test_consecutive_fiilters(self, triage_setup): collection = ImageFileCollection( location=triage_setup.test_dir, keywords=['imagetyp', 'filter', 'object']) no_files_match = collection.files_filtered(object='fdsafs') assert (len(no_files_match) == 0) some_files_should_match = collection.files_filtered(object=None, imagetyp='light') assert (len(some_files_should_match) == triage_setup.n_test['light'])
def test_initialization_with_no_keywords(self, triage_setup): # This test is primarily historical -- the old default for # keywords was an empty list (it is now the wildcard '*'). ic = ImageFileCollection(location=triage_setup.test_dir, keywords=[]) # iteration below failed before bugfix... execs = 0 for h in ic.headers(): execs += 1 assert not execs
def test_filter_files(self, triage_setup): img_collection = ImageFileCollection( location=triage_setup.test_dir, keywords=['imagetyp', 'filter']) assert len(img_collection.files_filtered( imagetyp='bias')) == triage_setup.n_test['bias'] assert len(img_collection.files) == triage_setup.n_test['files'] assert ('filter' in img_collection.keywords) assert ('flying monkeys' not in img_collection.keywords) assert len(img_collection.values('imagetyp', unique=True)) == 2
def test_headers(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp']) n_headers = 0 for header in collection.headers(): assert isinstance(header, fits.Header) assert ('bzero' in header) n_headers += 1 assert n_headers == triage_setup.n_test['files']
def test_setting_write_location_to_bad_dest_raises_error(self, tmpdir, triage_setup): new_tmp = tmpdir.mkdtemp() bad_directory = new_tmp.join('foo') ic = ImageFileCollection(triage_setup.test_dir, keywords=['naxis']) with pytest.raises(IOError): for hdr in ic.headers(save_location=bad_directory.strpath): pass
def test_summary_table_is_always_masked(self, triage_setup): # First, try grabbing all of the keywords ic = ImageFileCollection(location=triage_setup.test_dir, keywords='*') assert ic.summary.masked # Now, try keywords that every file will have ic.keywords = ['bitpix'] assert ic.summary.masked # What about keywords that include some that will surely be missing? ic.keywords = ['bitpix', 'dsafui'] assert ic.summary.masked
def test_filter_does_not_not_permanently_change_file_mask(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp']) # ensure all files are originally unmasked assert not collection.summary['file'].mask.any() # generate list that will match NO files collection.files_filtered(imagetyp='foisajfoisaj') # if the code works, this should have no permanent effect assert not collection.summary['file'].mask.any()
def test_headers_with_multiple_filters(self, triage_setup): collection = ImageFileCollection(location=triage_setup.test_dir, keywords=['imagetyp']) cnt = 0 for header in collection.headers(imagetyp='light', filter='R'): assert header['imagetyp'].lower() == 'light' assert header['filter'].lower() == 'r' cnt += 1 assert cnt == (triage_setup.n_test['light'] - triage_setup.n_test['missing_filter_value'])
def test_generator_with_regex(self, triage_setup): ic = ImageFileCollection(triage_setup.test_dir) n_light = 0 for h in ic.headers(regex_match=True, imagetyp='li.*t'): assert h['imagetyp'].lower() == 'light' n_light += 1 assert n_light == triage_setup.n_test['light']
def test_refresh_method_sees_added_files(self, triage_setup): ic = ImageFileCollection(triage_setup.test_dir, keywords='*') # Compressed files don't get copied. Not sure why... original_len = len(ic.summary) - triage_setup.n_test['compressed'] # Generate additional files in this directory for h in ic.headers(save_with_name="_foo"): pass ic.refresh() new_len = len(ic.summary) - triage_setup.n_test['compressed'] assert new_len == 2 * original_len
def test_regex_match_for_search(self, triage_setup): # Test regex matching in searches ic = ImageFileCollection(triage_setup.test_dir) files = ic.files_filtered(regex_match=True, imagetyp='b.*s') assert len(files) == triage_setup.n_test['bias'] # This should return all of the files in the test set all_files = ic.files_filtered(regex_match=True, imagetyp='bias|light') assert len(all_files) == triage_setup.n_test['files'] # Add a column with more interesting content and see whether we # match that. ic.summary['match_me'] = [ 'hello', 'goodbye', 'bye', 'byte', 'good bye hello', 'dog' ] hello_anywhere = ic.files_filtered(regex_match=True, match_me='hello') assert len(hello_anywhere) == 2 hello_start = ic.files_filtered(regex_match=True, match_me='^hello') assert len(hello_start) == 1 # Is it really a case-insensitive match? hello_start = ic.files_filtered(regex_match=True, match_me='^HeLlo') assert len(hello_start) == 1 any_bye = ic.files_filtered(regex_match=True, match_me='by.*e') assert len(any_bye) == 4
def test_filtered_files_have_proper_path(self, triage_setup): ic = ImageFileCollection(location=triage_setup.test_dir, keywords='*') # Get a subset of the files. plain_biases = ic.files_filtered(imagetyp='bias') # Force a copy... plain_biases = list(plain_biases) # Same subset, but with full path. path_biases = ic.files_filtered(imagetyp='bias', include_path=True) for path_b, plain_b in zip(path_biases, plain_biases): # If the path munging has been done properly, this will succeed. assert os.path.basename(path_b) == plain_b
def test_fits_summary_when_keywords_are_not_subset(self, triage_setup): """ Catch case when there is overlap between keyword list passed to the ImageFileCollection and to files_filtered but the latter is not a subset of the former. """ ic = ImageFileCollection(triage_setup.test_dir, keywords=['imagetyp', 'exposure']) n_files = len(ic.files) files_missing_this_key = ic.files_filtered(imagetyp='*', monkeys=None) assert (n_files > 0) assert (n_files == len(files_missing_this_key))
def test_dir_with_no_fits_files(self, tmpdir): empty_dir = tmpdir.mkdtemp() some_file = empty_dir.join('some_file.txt') some_file.dump('words') with catch_warnings() as w: collection = ImageFileCollection(location=empty_dir.strpath, keywords=['imagetyp']) assert len(w) == 1 assert str(w[0].message) == "no FITS files in the collection." assert collection.summary is None for hdr in collection.headers(): # this statement should not be reached if there are no FITS files assert 0
def test_fits_summary(triage_setup): keywords = ['imagetyp', 'filter'] ic = ImageFileCollection(triage_setup.test_dir, keywords=keywords) summary = ic._fits_summary(header_keywords=keywords) assert len(summary['file']) == triage_setup.n_test['files'] for keyword in keywords: assert len(summary[keyword]) == triage_setup.n_test['files'] # explicit conversion to array is needed to avoid astropy Table bug in # 0.2.4 no_filter_no_object_row = np.array( summary['file'] == 'no_filter_no_object_bias.fit') # there should be no filter keyword in the bias file assert summary['filter'][no_filter_no_object_row].mask
def test_make_collection_by_filtering(self, triage_setup): # Test for implementation of feature at # # https://github.com/astropy/ccdproc/issues/596 # # which adds the ability to create a new collection by filtering # an existing ImageFileCollection. ic = ImageFileCollection(location=triage_setup.test_dir) new_ic = ic.filter(imagetyp='light') assert len(new_ic.summary) == triage_setup.n_test['light'] for header in new_ic.headers(): assert header['imagetyp'].lower() == 'light'
def test_refresh_method_sees_added_keywords(self, triage_setup): ic = ImageFileCollection(triage_setup.test_dir, keywords='*') # Add a keyword I know isn't already in the header to each file. not_in_header = 'BARKARK' for h in ic.headers(overwrite=True): h[not_in_header] = True assert not_in_header not in ic.summary.colnames ic.refresh() # After refreshing the odd keyword should be present. assert not_in_header.lower() in ic.summary.colnames
def test_filter_files_whitespace_keys(self, triage_setup): hdr = fits.Header([('HIERARCH a b', 2)]) hdul = fits.HDUList([fits.PrimaryHDU(np.ones((10, 10)), header=hdr)]) hdul.writeto( os.path.join(triage_setup.test_dir, 'hdr_with_whitespace.fits')) ic = ImageFileCollection(location=triage_setup.test_dir) # Using a dictionary and unpacking it should work filtered = ic.files_filtered(**{'a b': 2}) assert len(filtered) == 1 assert 'hdr_with_whitespace.fits' in filtered # Also check it's working with generators: for _, filename in ic.data(a_b=2, replace_='_', return_fname=True): assert filename == 'hdr_with_whitespace.fits'
def test_hdus(self, triage_setup): collection = ImageFileCollection( location=triage_setup.test_dir, keywords=['imagetyp']) n_hdus = 0 for hdu in collection.hdus(): assert isinstance(hdu, fits.PrimaryHDU) data = hdu.data # must access the data to force scaling # pre-astropy 1.1 unsigned data was changed to float32 and BZERO # removed. In 1.1 and later, BZERO stays but the data type is # unsigned int. assert (('BZERO' not in hdu.header) or (data.dtype is np.dtype(np.uint16))) n_hdus += 1 assert n_hdus == triage_setup.n_test['files']
def test_force_detect_fits_files_finds_fits_files(self, triage_setup): # Tests for new feature # # https://github.com/astropy/ccdproc/issues/620 # # which supports adding all of the FITS files in a location based on # their *contents* instead of their *extension*. # Grab files from the default collection and make a copy with a new name # (and with no fits-like extension) # # Making a copy of *every* file means we can just double the expected # number of files as part of the tests. path = Path(triage_setup.test_dir) for idx, p in enumerate(path.iterdir()): new_name = 'no_extension{}'.format(idx) new_path = path / new_name new_path.write_bytes(p.read_bytes()) ic = ImageFileCollection(location=str(path), find_fits_by_reading=True) # Compressed files won't be automagically detected by reading the # first few bytes. expected_number = (2 * triage_setup.n_test['files'] - triage_setup.n_test['compressed']) assert len(ic.summary) == expected_number n_bias = (ic.summary['imagetyp'] == 'BIAS').sum() assert n_bias == 2 * triage_setup.n_test['bias'] # Only one file in the original set of test files has exposure time # 15, so there should be two now. assert len(ic.files_filtered(exposure=15.0)) == 2 # Try one of the generators expected_heads = (2 * triage_setup.n_test['light'] - triage_setup.n_test['compressed']) n_heads = 0 for h in ic.headers(imagetyp='light'): assert h['imagetyp'].lower() == 'light' n_heads += 1 assert n_heads == expected_heads
def test_duplicate_keywords_in_setting(self, triage_setup): keywords_in = ['imagetyp', 'a', 'a'] ic = ImageFileCollection(triage_setup.test_dir, keywords=keywords_in) for key in set(keywords_in): assert (key in ic.keywords) # one keyword gets added: file assert len(ic.keywords) < len(keywords_in) + 1
def test_keyword_includes_file(self, triage_setup): keywords_in = ['file', 'imagetyp'] ic = ImageFileCollection(triage_setup.test_dir, keywords=keywords_in) assert 'file' in ic.keywords file_keywords = [key for key in ic.keywords if key == 'file'] assert len(file_keywords) == 1
def test_repr_globs(self, triage_setup): ic = ImageFileCollection(location=triage_setup.test_dir, glob_exclude="*no_filter*", glob_include="*object_light*") ref = ("ImageFileCollection(location={0!r}, " "glob_include='*object_light*', " "glob_exclude='*no_filter*')".format(triage_setup.test_dir)) assert repr(ic) == ref
def test_collection_when_one_file_not_fits(self, triage_setup): not_fits = 'foo.fit' path_bad = os.path.join(triage_setup.test_dir, not_fits) # create an empty file... with open(path_bad, 'w'): pass ic = ImageFileCollection(triage_setup.test_dir, keywords=['imagetyp']) assert not_fits not in ic.summary['file'] os.remove(path_bad)