def _process_spectrum_from_fits(self, data_product):
flux, header = fits.getdata(data_product.data.name, header=True)
try:
flux_constant = Unit(header['BUNIT'])
flux_constant.to(DEFAULT_FLUX_CONSTANT)
except:
flux_constant = DEFAULT_FLUX_CONSTANT
date_obs = header.get('DATE-OBS', datetime.now())
header['CUNIT1'] = 'Angstrom'
wcs = WCS(header=header)
flux = flux[0].squeeze() * flux_constant
spectrum = Spectrum1D(flux=flux, wcs=wcs)
return spectrum, Time(date_obs).to_datetime()
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in root.attrs.iteritems():
typ = key[0]
if typ == 'tag':
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == 'fitsheaderentry':
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(not_(and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion))))
else:
query = query.filter(and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion)))
elif typ == 'download time':
start, end, inverted = value
if inverted:
query = query.filter(
~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(
DatabaseEntry.download_time.between(start, end))
elif typ == 'path':
path, inverted = value
if inverted:
query = query.filter(or_(
DatabaseEntry.path != path, DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == 'wave':
min_, max_, unit = value
waveunit = Unit(unit)
# convert min_ and max_ to nm from the unit `waveunit`
wavemin = waveunit.to(nm, min_, equivalencies.spectral())
wavemax = waveunit.to(nm, max_, equivalencies.spectral())
query = query.filter(and_(
DatabaseEntry.wavemin >= wavemin,
DatabaseEntry.wavemax <= wavemax))
elif typ == 'time':
start, end, near = value
query = query.filter(and_(
DatabaseEntry.observation_time_start < end,
DatabaseEntry.observation_time_end > start))
else:
query = query.filter_by(**{typ: value})
return query.all()
def entries_from_file(file, default_waveunit=None):
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE))
>>> len(entries)
1
>>> entry = entries.pop()
>>> entry.instrument
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2012, 1, 1, 0, 16, 7, 836000), None)
>>> entry.wavemin, entry.wavemax
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries)
112
"""
headers = fits.get_header(file)
if isinstance(file, (str, unicode)):
filename = file
else:
filename = getattr(file, "name", None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in header.iteritems():
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == "":
value = str(value)
elif key == "KEYCOMMENTS":
for k, v in value.iteritems():
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == "INSTRUME":
entry.instrument = value
elif key == "WAVELNTH":
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ("DATE-END", "DATE_END"):
entry.observation_time_end = parse_time(value)
elif key in ("DATE-OBS", "DATE_OBS"):
entry.observation_time_start = parse_time(value)
yield entry
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> qr = client.query(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit'))
>>> entry = DatabaseEntry.from_query_result_block(qr[0])
>>> entry.source
'SOHO'
>>> entry.provider
'SDAC'
>>> entry.physobs
'intensity'
>>> entry.fileid
'/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.010014'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2001, 1, 1, 1, 0, 14), datetime.datetime(2001, 1, 1, 1, 0, 21))
>>> entry.instrument
'EIT'
>>> entry.size
2059.0
>>> entry.wavemin, entry.wavemax
(17.1, 17.1)
"""
time_start = timestamp2datetime("%Y%m%d%H%M%S", qr_block.time.start)
time_end = timestamp2datetime("%Y%m%d%H%M%S", qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but AstroPy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin), equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax), equivalencies.spectral())
source = str(qr_block.source) if qr_block.source is not None else None
provider = str(qr_block.provider) if qr_block.provider is not None else None
fileid = str(qr_block.fileid) if qr_block.fileid is not None else None
instrument = str(qr_block.instrument) if qr_block.instrument is not None else None
physobs = getattr(qr_block, "physobs", None)
if physobs is not None:
physobs = str(physobs)
return cls(
source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=qr_block.size,
wavemin=wavemin,
wavemax=wavemax,
)
def entries_from_file(file,
default_waveunit=None,
time_string_parse_format=None):
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
time_string_parse_format : str, optional
Fallback timestamp format which will be passed to
`~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to
automatically read the `date-obs` metadata.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> from sunpy.database.tables import entries_from_file
>>> import sunpy.data.sample # doctest: +REMOTE_DATA
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA
>>> len(entries) # doctest: +REMOTE_DATA
1
>>> entry = entries.pop() # doctest: +REMOTE_DATA
>>> entry.instrument # doctest: +REMOTE_DATA
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None)
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA
111
"""
headers = fits.get_header(file)
if isinstance(file, (str, six.text_type)):
filename = file
else:
filename = getattr(file, 'name', None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in six.iteritems(header):
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == '':
value = str(value)
elif key == 'KEYCOMMENTS':
for k, v in six.iteritems(value):
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
entry.hdu_index = headers.index(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == 'INSTRUME':
entry.instrument = value
elif key == 'WAVELNTH':
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(
nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ('DATE-END', 'DATE_END'):
entry.observation_time_end = parse_time(
value, _time_string_parse_format=time_string_parse_format)
elif key in ('DATE-OBS', 'DATE_OBS'):
entry.observation_time_start = parse_time(
value, _time_string_parse_format=time_string_parse_format)
yield entry
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None):
"""
Make a new :class:`DatabaseEntry` instance from a Fido search
result block.
Parameters
----------
sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock`
A query result block is usually not created directly; instead,
one gets instances of
``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating
over each element of a Fido search result.
default_waveunit : `str`, optional
The wavelength unit that is used if it cannot be found in the
`sr_block`.
"""
# All attributes of DatabaseEntry that are not in QueryResponseBlock
# are set as None for now.
source = getattr(sr_block, 'source', None)
provider = getattr(sr_block, 'provider', None)
physobs = getattr(sr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
instrument = getattr(sr_block, 'instrument', None)
time_start = sr_block.time.start
time_end = sr_block.time.end
wavelengths = getattr(sr_block, 'wave', None)
wavelength_temp = {}
if isinstance(wavelength_temp, tuple):
# Tuple of values
wavelength_temp['wavemin'] = wavelengths[0]
wavelength_temp['wavemax'] = wavelengths[1]
else:
# Single Value
wavelength_temp['wavemin'] = wavelength_temp[
'wavemax'] = wavelengths
final_values = {}
for key, val in wavelength_temp.items():
if isinstance(val, quantity.Quantity):
unit = getattr(val, 'unit', None)
if unit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
raise WaveunitNotFoundError(sr_block)
final_values[key] = unit.to(nm, float(val.value),
equivalencies.spectral())
elif val is None or np.isnan(val):
final_values[key] = val
wavemin = final_values['wavemin']
wavemax = final_values['wavemax']
# sr_block.url of a QueryResponseBlock attribute is stored in fileid
fileid = str(sr_block.url) if sr_block.url is not None else None
size = None
return cls(source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=size,
wavemin=wavemin,
wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> from sunpy.database.tables import DatabaseEntry
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.search(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit')) # doctest: +REMOTE_DATA
>>> entry = DatabaseEntry._from_query_result_block(qr[0]) # doctest: +REMOTE_DATA
>>> entry.source # doctest: +REMOTE_DATA
SOHO
>>> entry.provider # doctest: +REMOTE_DATA
SDAC
>>> entry.physobs # doctest: +REMOTE_DATA
'intensity'
>>> entry.fileid # doctest: +REMOTE_DATA
/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.000042
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2001, 1, 1, 0, 0, 42), datetime.datetime(2001, 1, 1, 0, 0, 54))
>>> entry.instrument # doctest: +REMOTE_DATA
EIT
>>> entry.size # doctest: +REMOTE_DATA
2059.0
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(19.5, 19.5)
"""
time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start)
if not qr_block.time.end:
qr_block.time.end = qr_block.time.start
time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but Astropy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin),
equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax),
equivalencies.spectral())
source = getattr(qr_block, 'source', None)
provider = getattr(qr_block, 'provider', None)
fileid = getattr(qr_block, 'fileid', None)
instrument = getattr(qr_block, 'instrument', None)
size = getattr(qr_block, 'size', -1)
physobs = getattr(qr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
return cls(source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=size,
wavemin=wavemin,
wavemax=wavemax)
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None):
"""
Make a new :class:`DatabaseEntry` instance from a Fido search
result block.
Parameters
----------
sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock`
A query result block is usually not created directly; instead,
one gets instances of
``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating
over each element of a Fido search result.
default_waveunit : `str`, optional
The wavelength unit that is used if it cannot be found in the
`sr_block`.
Examples
--------
>>> from sunpy.net import Fido, attrs
>>> from sunpy.database.tables import DatabaseEntry
>>> sr = Fido.search(attrs.Time("2012/1/1", "2012/1/2"),
... attrs.Instrument('lyra'))
>>> entry = DatabaseEntry._from_fido_search_result_block(sr[0][0])
>>> entry.source
'Proba2'
>>> entry.provider
'esa'
>>> entry.physobs
'irradiance'
>>> entry.fileid
'http://proba2.oma.be/lyra/data/bsd/2012/01/01/lyra_20120101-000000_lev2_std.fits'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2012, 1, 1, 0, 0), datetime.datetime(2012, 1, 2, 0, 0))
>>> entry.instrument
'lyra'
"""
# All attributes of DatabaseEntry that are not in QueryResponseBlock
# are set as None for now.
source = getattr(sr_block, 'source', None)
provider = getattr(sr_block, 'provider', None)
physobs = getattr(sr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
instrument = getattr(sr_block, 'instrument', None)
time_start = sr_block.time.start
time_end = sr_block.time.end
wavelengths = getattr(sr_block, 'wave', None)
wavelength_temp = {}
if isinstance(wavelength_temp, tuple):
# Tuple of values
wavelength_temp['wavemin'] = wavelengths[0]
wavelength_temp['wavemax'] = wavelengths[1]
else:
# Single Value
wavelength_temp['wavemin'] = wavelength_temp['wavemax'] = wavelengths
final_values = {}
for key, val in wavelength_temp.items():
if isinstance(val, quantity.Quantity):
unit = getattr(val, 'unit', None)
if unit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
raise WaveunitNotFoundError(sr_block)
final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral())
elif val is None or np.isnan(val):
final_values[key] = val
wavemin = final_values['wavemin']
wavemax = final_values['wavemax']
# sr_block.url of a QueryResponseBlock attribute is stored in fileid
fileid = str(sr_block.url) if sr_block.url is not None else None
size = None
return cls(
source=source, provider=provider, physobs=physobs, fileid=fileid,
observation_time_start=time_start, observation_time_end=time_end,
instrument=instrument, size=size,
wavemin=wavemin, wavemax=wavemax)
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in root.attrs.iteritems():
typ = key[0]
if typ == "tag":
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == "fitsheaderentry":
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(
not_(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion),
)
)
)
else:
query = query.filter(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion),
)
)
elif typ == "download time":
start, end, inverted = value
if inverted:
query = query.filter(~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(DatabaseEntry.download_time.between(start, end))
elif typ == "path":
path, inverted = value
if inverted:
query = query.filter(or_(DatabaseEntry.path != path, DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == "wave":
min_, max_, unit = value
waveunit = Unit(unit)
# convert min_ and max_ to nm from the unit `waveunit`
wavemin = waveunit.to(nm, min_, equivalencies.spectral())
wavemax = waveunit.to(nm, max_, equivalencies.spectral())
query = query.filter(and_(DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax))
elif typ == "time":
start, end, near = value
query = query.filter(
and_(DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start)
)
else:
if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(SUPPORTED_NONVSO_ATTRS):
raise NotImplementedError("The attribute {0!r} is not yet supported to query a database.".format(typ))
query = query.filter_by(**{typ: value})
return query.all()
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None):
"""
Make a new :class:`DatabaseEntry` instance from a Fido search
result block.
Parameters
----------
sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock`
A query result block is usually not created directly; instead,
one gets instances of
``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating
over each element of a Fido search result.
default_waveunit : `str`, optional
The wavelength unit that is used if it cannot be found in the
`sr_block`.
"""
# All attributes of DatabaseEntry that are not in QueryResponseBlock
# are set as None for now.
source = getattr(sr_block, 'source', None)
provider = getattr(sr_block, 'provider', None)
physobs = getattr(sr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
instrument = getattr(sr_block, 'instrument', None)
time_start = sr_block.time.start
time_end = sr_block.time.end
wavelengths = getattr(sr_block, 'wave', None)
wavelength_temp = {}
if isinstance(wavelength_temp, tuple):
# Tuple of values
wavelength_temp['wavemin'] = wavelengths[0]
wavelength_temp['wavemax'] = wavelengths[1]
else:
# Single Value
wavelength_temp['wavemin'] = wavelength_temp['wavemax'] = wavelengths
final_values = {}
for key, val in wavelength_temp.items():
if isinstance(val, quantity.Quantity):
unit = getattr(val, 'unit', None)
if unit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
raise WaveunitNotFoundError(sr_block)
final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral())
elif val is None or np.isnan(val):
final_values[key] = val
wavemin = final_values['wavemin']
wavemax = final_values['wavemax']
# sr_block.url of a QueryResponseBlock attribute is stored in fileid
fileid = str(sr_block.url) if sr_block.url is not None else None
size = None
return cls(
source=source, provider=provider, physobs=physobs, fileid=fileid,
observation_time_start=time_start, observation_time_end=time_end,
instrument=instrument, size=size,
wavemin=wavemin, wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> from sunpy.database.tables import DatabaseEntry
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.search(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit')) # doctest: +REMOTE_DATA
>>> entry = DatabaseEntry._from_query_result_block(qr[0]) # doctest: +REMOTE_DATA
>>> entry.source # doctest: +REMOTE_DATA
SOHO
>>> entry.provider # doctest: +REMOTE_DATA
SDAC
>>> entry.physobs # doctest: +REMOTE_DATA
'intensity'
>>> entry.fileid # doctest: +REMOTE_DATA
/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.000042
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2001, 1, 1, 0, 0, 42), datetime.datetime(2001, 1, 1, 0, 0, 54))
>>> entry.instrument # doctest: +REMOTE_DATA
EIT
>>> entry.size # doctest: +REMOTE_DATA
2059.0
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(19.5, 19.5)
"""
time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start)
if not qr_block.time.end:
qr_block.time.end = qr_block.time.start
time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but Astropy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin),
equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax),
equivalencies.spectral())
source = getattr(qr_block, 'source', None)
provider = getattr(qr_block, 'provider', None)
fileid = getattr(qr_block, 'fileid', None)
instrument = getattr(qr_block, 'instrument', None)
size = getattr(qr_block, 'size', -1)
physobs = getattr(qr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
return cls(
source=source, provider=provider, physobs=physobs, fileid=fileid,
observation_time_start=time_start, observation_time_end=time_end,
instrument=instrument, size=size,
wavemin=wavemin, wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> qr = client.query(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit'))
>>> entry = DatabaseEntry.from_query_result_block(qr[0])
>>> entry.source
'SOHO'
>>> entry.provider
'SDAC'
>>> entry.physobs
'intensity'
>>> entry.fileid
'/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.010014'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2001, 1, 1, 1, 0, 14), datetime.datetime(2001, 1, 1, 1, 0, 21))
>>> entry.instrument
'EIT'
>>> entry.size
2059.0
>>> entry.wavemin, entry.wavemax
(17.1, 17.1)
"""
time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start)
time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but AstroPy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin),
equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax),
equivalencies.spectral())
source = str(qr_block.source) if qr_block.source is not None else None
provider = str(
qr_block.provider) if qr_block.provider is not None else None
fileid = str(qr_block.fileid) if qr_block.fileid is not None else None
instrument = str(
qr_block.instrument) if qr_block.instrument is not None else None
physobs = getattr(qr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
return cls(source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=qr_block.size,
wavemin=wavemin,
wavemax=wavemax)
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in root.attrs.iteritems():
typ = key[0]
if typ == 'tag':
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == 'fitsheaderentry':
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(
not_(
and_(
DatabaseEntry.fits_header_entries.any(
key_criterion),
DatabaseEntry.fits_header_entries.any(
value_criterion))))
else:
query = query.filter(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(
value_criterion)))
elif typ == 'download time':
start, end, inverted = value
if inverted:
query = query.filter(
~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(
DatabaseEntry.download_time.between(start, end))
elif typ == 'path':
path, inverted = value
if inverted:
query = query.filter(
or_(DatabaseEntry.path != path,
DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == 'wave':
min_, max_, unit = value
waveunit = Unit(unit)
# convert min_ and max_ to nm from the unit `waveunit`
wavemin = waveunit.to(nm, min_, equivalencies.spectral())
wavemax = waveunit.to(nm, max_, equivalencies.spectral())
query = query.filter(
and_(DatabaseEntry.wavemin >= wavemin,
DatabaseEntry.wavemax <= wavemax))
elif typ == 'time':
start, end, near = value
query = query.filter(
and_(DatabaseEntry.observation_time_start < end,
DatabaseEntry.observation_time_end > start))
else:
if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(
SUPPORTED_NONVSO_ATTRS):
raise NotImplementedError(
"The attribute {0!r} is not yet supported to query a database."
.format(typ))
query = query.filter_by(**{typ: value})
return query.all()
def entries_from_file(file, default_waveunit=None,
time_string_parse_format=''):
# Note: time_string_parse_format='' so that None won't be passed to Time.strptime
# (which would make strptime freak out, if I remember correctly).
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
time_string_parse_format : str, optional
Fallback timestamp format which will be passed to
`~astropy.time.Time.strptime` if `sunpy.time.parse_time` is unable to
automatically read the `date-obs` metadata.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> from sunpy.database.tables import entries_from_file
>>> import sunpy.data.sample # doctest: +REMOTE_DATA
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA
>>> len(entries) # doctest: +REMOTE_DATA
1
>>> entry = entries.pop() # doctest: +REMOTE_DATA
>>> entry.instrument # doctest: +REMOTE_DATA
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None)
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA
111
"""
headers = fits.get_header(file)
# This just checks for blank default headers
# due to compression.
for header in headers:
if header == DEFAULT_HEADER:
headers.remove(header)
if isinstance(file, str):
filename = file
else:
filename = getattr(file, 'name', None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in header.items():
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == '':
value = str(value)
elif key == 'KEYCOMMENTS':
for k, v in value.items():
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
entry.hdu_index = headers.index(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == 'INSTRUME':
entry.instrument = value
elif key == 'WAVELNTH':
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(
nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ('DATE-END', 'DATE_END'):
try:
dt = parse_time(value).datetime
except ValueError:
dt = Time.strptime(value, time_string_parse_format).datetime
entry.observation_time_end = dt
elif key in ('DATE-OBS', 'DATE_OBS'):
try:
dt = parse_time(value).datetime
except ValueError:
dt = Time.strptime(value, time_string_parse_format).datetime
entry.observation_time_start = dt
yield entry
