def test_extract_waveunit(fname, waveunit, warn): if warn: with pytest.warns(AstropyUserWarning, match='File may have been truncated'): waveunit = extract_waveunit(get_header(fname)[0]) else: waveunit = extract_waveunit(get_header(fname)[0]) assert waveunit is waveunit
def num_entries_from_vso_query(db, query, path=None, file_pattern='', overwrite=False): db.download_from_vso_query_result( query, path=path, overwrite=overwrite) fits_pattern = file_pattern num_of_fits_headers = sum( len(fits.get_header(file)) for file in glob.glob(fits_pattern)) return num_of_fits_headers
def _restore_from_fits(self, filename): """ Helper to load cube from FITS file """ tmp = astropy.io.fits.open(filename) header = MetaDict(get_header(tmp)[0]) data = tmp[0].data*u.Unit(header['bunit']) wavelength = tmp[1].data.field(0)*u.Unit(tmp[1].header['TUNIT1']) tmp.close() return np.swapaxes(data.T, 0, 1), header, wavelength
def test_download_from_qr(database, download_qr, tmpdir): assert len(database) == 0 database.download_from_vso_query_result(download_qr, path=str(tmpdir.join("{file}.fits"))) fits_pattern = str(tmpdir.join("*.fits")) num_of_fits_headers = sum(len(fits.get_header(file)) for file in glob.glob(fits_pattern)) assert len(database) == num_of_fits_headers > 0 for entry in database: assert os.path.dirname(entry.path) == str(tmpdir) database.undo() assert len(database) == 0 database.redo() assert len(database) == num_of_fits_headers > 0
def test_download(database, download_query, tmpdir): assert len(database) == 0 database.default_waveunit = "angstrom" database.download(*download_query, path=str(tmpdir.join("{file}.fits")), progress=True) fits_pattern = str(tmpdir.join("*.fits")) num_of_fits_headers = sum(len(fits.get_header(file)) for file in glob.glob(fits_pattern)) assert len(database) == num_of_fits_headers for entry in database: assert os.path.dirname(entry.path) == str(tmpdir) database.undo() assert len(database) == 0 database.redo() assert len(database) == 4
def test_download_from_qr(database, download_qr, tmpdir): assert len(database) == 0 database.download_from_vso_query_result( download_qr, path=str(tmpdir.join('{file}.fits'))) fits_pattern = str(tmpdir.join('*.fits')) num_of_fits_headers = sum( len(fits.get_header(file)) for file in glob.glob(fits_pattern)) assert len(database) == num_of_fits_headers > 0 for entry in database: assert os.path.dirname(entry.path) == str(tmpdir) database.undo() assert len(database) == 0 database.redo() assert len(database) == num_of_fits_headers > 0
def test_fetch(database, download_query, tmpdir): assert len(database) == 0 database.default_waveunit = 'angstrom' database.fetch( *download_query, path=str(tmpdir.join('{file}.fits')), progress=True) fits_pattern = str(tmpdir.join('*.fits')) num_of_fits_headers = sum( len(fits.get_header(file)) for file in glob.glob(fits_pattern)) assert len(database) == num_of_fits_headers for entry in database: assert os.path.dirname(entry.path) == str(tmpdir) database.undo() assert len(database) == 0 database.redo() assert len(database) == 4
def test_fetch(database, download_query, tmpdir): assert len(database) == 0 database.default_waveunit = 'angstrom' database.fetch( *download_query, path=str(tmpdir.join('{file}.fits')), progress=True) fits_pattern = str(tmpdir.join('*.fits')) num_of_fits_headers = sum( len(fits.get_header(file)) for file in glob.glob(fits_pattern)) assert len(database) == num_of_fits_headers for entry in database: assert os.path.dirname(entry.path) == str(tmpdir) database.undo() assert len(database) == 0 database.redo() # Make this resilitent to vso changes while we chase this up with VSO 2018-03-07 assert len(database) in (2, 4)
def test_extract_waveunit_wavelnthcomment_parentheses(): # WAVELNTH comment is: "Observed wavelength (nm)" with pytest.warns(AstropyUserWarning, match='File may have been truncated'): waveunit = extract_waveunit(get_header(SVSM_IMAGE)[0]) assert waveunit == 'nm'
def test_extract_waveunit_minus10(): # value of WAVEUNIT is -10 with pytest.warns(AstropyUserWarning, match='File may have been truncated'): waveunit = extract_waveunit(get_header(MQ_IMAGE)[0]) assert waveunit == 'angstrom'
def test_extract_waveunit_wavelnthcomment_parentheses(): # WAVELNTH comment is: "Observed wavelength (nm)" waveunit = extract_waveunit(get_header(SVSM_IMAGE)[0]) assert waveunit == 'nm'
def test_extract_waveunit_wavelnthcomment_brackets(): # WAVELNTH comment is: "[Angstrom] bandpass peak response" waveunit = extract_waveunit(get_header(SWAP_LEVEL1_IMAGE)[0]) assert waveunit == 'angstrom'
def test_extract_waveunit_waveunitcomment(): # comment of WAVEUNIT is: "in meters" with pytest.warns(AstropyUserWarning, match='File may have been truncated'): waveunit = extract_waveunit(get_header(NA_IMAGE)[0]) assert waveunit == 'm'
def entries_from_file(file, default_waveunit=None, time_string_parse_format=None): """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. time_string_parse_format : str, optional Fallback timestamp format which will be passed to `~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to automatically read the `date-obs` metadata. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> from sunpy.database.tables import entries_from_file >>> import sunpy.data.sample # doctest: +REMOTE_DATA >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA >>> len(entries) # doctest: +REMOTE_DATA 1 >>> entry = entries.pop() # doctest: +REMOTE_DATA >>> entry.instrument # doctest: +REMOTE_DATA 'SWAP' >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None) >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA 111 """ headers = fits.get_header(file) if isinstance(file, (str, six.text_type)): filename = file else: filename = getattr(file, 'name', None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in six.iteritems(header): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == '': value = str(value) elif key == 'KEYCOMMENTS': for k, v in six.iteritems(value): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) entry.hdu_index = headers.index(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == 'INSTRUME': entry.instrument = value elif key == 'WAVELNTH': if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to( nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ('DATE-END', 'DATE_END'): entry.observation_time_end = parse_time( value, _time_string_parse_format=time_string_parse_format) elif key in ('DATE-OBS', 'DATE_OBS'): entry.observation_time_start = parse_time( value, _time_string_parse_format=time_string_parse_format) yield entry
def test_extract_waveunit_minus10(): # value of WAVEUNIT is -10 waveunit = extract_waveunit(get_header(MQ_IMAGE)[0]) assert waveunit == 'angstrom'
def test_extract_waveunit_from_waveunit_key(): # the key WAVEUNIT can be accessed and returned directly waveunit = extract_waveunit(get_header(AIA_171_IMAGE)[0]) assert waveunit == 'angstrom'
def test_extract_waveunit_missing_waveunit_key_and_missing_wavelnth_comment(): waveunit = extract_waveunit(get_header(RHESSI_IMAGE)[0]) assert waveunit is None
def test_missing_waveunit_in_wavelnth_comment(): # the comment of the key WAVELNTH has the value # '171 = Fe IX/X, 195 = Fe XII,' which contains no unit information waveunit = extract_waveunit(get_header(EIT_195_IMAGE)[0]) assert waveunit is None
def test_extract_waveunit_minus9(): # value of WAVEUNIT is -9 waveunit = extract_waveunit(get_header(MEDN_IMAGE)[0]) assert waveunit == 'nm'
def test_extract_waveunit_waveunitcomment(): # comment of WAVEUNIT is: "in meters" waveunit = extract_waveunit(get_header(NA_IMAGE)[0]) assert waveunit == 'm'
def entries_from_file(file, default_waveunit=None): """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) >>> len(entries) 1 >>> entry = entries.pop() >>> entry.instrument 'SWAP' >>> entry.observation_time_start, entry.observation_time_end (datetime.datetime(2012, 1, 1, 0, 16, 7, 836000), None) >>> entry.wavemin, entry.wavemax (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) 112 """ headers = fits.get_header(file) if isinstance(file, (str, unicode)): filename = file else: filename = getattr(file, "name", None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in header.iteritems(): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == "": value = str(value) elif key == "KEYCOMMENTS": for k, v in value.iteritems(): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == "INSTRUME": entry.instrument = value elif key == "WAVELNTH": if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to(nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ("DATE-END", "DATE_END"): entry.observation_time_end = parse_time(value) elif key in ("DATE-OBS", "DATE_OBS"): entry.observation_time_start = parse_time(value) yield entry
def entries_from_file(file, default_waveunit=None, time_string_parse_format=''): # Note: time_string_parse_format='' so that None won't be passed to Time.strptime # (which would make strptime freak out, if I remember correctly). """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. time_string_parse_format : str, optional Fallback timestamp format which will be passed to `~astropy.time.Time.strptime` if `sunpy.time.parse_time` is unable to automatically read the `date-obs` metadata. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> from sunpy.database.tables import entries_from_file >>> import sunpy.data.sample # doctest: +REMOTE_DATA >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA >>> len(entries) # doctest: +REMOTE_DATA 1 >>> entry = entries.pop() # doctest: +REMOTE_DATA >>> entry.instrument # doctest: +REMOTE_DATA 'SWAP' >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None) >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA 111 """ headers = fits.get_header(file) # This just checks for blank default headers # due to compression. for header in headers: if header == DEFAULT_HEADER: headers.remove(header) if isinstance(file, str): filename = file else: filename = getattr(file, 'name', None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in header.items(): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == '': value = str(value) elif key == 'KEYCOMMENTS': for k, v in value.items(): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) entry.hdu_index = headers.index(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == 'INSTRUME': entry.instrument = value elif key == 'WAVELNTH': if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to( nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ('DATE-END', 'DATE_END'): try: dt = parse_time(value).datetime except ValueError: dt = Time.strptime(value, time_string_parse_format).datetime entry.observation_time_end = dt elif key in ('DATE-OBS', 'DATE_OBS'): try: dt = parse_time(value).datetime except ValueError: dt = Time.strptime(value, time_string_parse_format).datetime entry.observation_time_start = dt yield entry