def get_equivalencies():
"""
Return a list of example equivalencies for testing serialization.
"""
return [
eq.plate_scale(.3 * u.deg / u.mm),
eq.pixel_scale(.5 * u.deg / u.pix),
eq.spectral_density(350 * u.nm, factor=2),
eq.spectral_density(350 * u.nm),
eq.spectral(),
eq.brightness_temperature(500 * u.GHz),
eq.brightness_temperature(500 * u.GHz, beam_area=23 * u.sr),
eq.with_H0(),
eq.temperature_energy(),
eq.temperature(),
eq.thermodynamic_temperature(300 * u.Hz),
eq.thermodynamic_temperature(140 * u.GHz, Planck15.Tcmb0),
eq.beam_angular_area(3 * u.sr),
eq.mass_energy(),
eq.molar_mass_amu(),
eq.doppler_relativistic(2 * u.m),
eq.doppler_optical(2 * u.nm),
eq.doppler_radio(2 * u.Hz),
eq.parallax(),
eq.logarithmic(),
eq.dimensionless_angles(),
eq.spectral() + eq.temperature(),
(eq.spectral_density(35 * u.nm) +
eq.brightness_temperature(5 * u.Hz, beam_area=2 * u.sr)),
(eq.spectral() + eq.spectral_density(35 * u.nm) +
eq.brightness_temperature(5 * u.Hz, beam_area=2 * u.sr))
]
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in root.attrs.iteritems():
typ = key[0]
if typ == 'tag':
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == 'fitsheaderentry':
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(not_(and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion))))
else:
query = query.filter(and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion)))
elif typ == 'download time':
start, end, inverted = value
if inverted:
query = query.filter(
~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(
DatabaseEntry.download_time.between(start, end))
elif typ == 'path':
path, inverted = value
if inverted:
query = query.filter(or_(
DatabaseEntry.path != path, DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == 'wave':
min_, max_, unit = value
waveunit = Unit(unit)
# convert min_ and max_ to nm from the unit `waveunit`
wavemin = waveunit.to(nm, min_, equivalencies.spectral())
wavemax = waveunit.to(nm, max_, equivalencies.spectral())
query = query.filter(and_(
DatabaseEntry.wavemin >= wavemin,
DatabaseEntry.wavemax <= wavemax))
elif typ == 'time':
start, end, near = value
query = query.filter(and_(
DatabaseEntry.observation_time_start < end,
DatabaseEntry.observation_time_end > start))
else:
query = query.filter_by(**{typ: value})
return query.all()
def test_equivalencies():
model = UnitsMapping(((u.m, u.dimensionless_unscaled), ))
with pytest.raises(UnitsError):
model(Quantity(100, u.Hz))
model = UnitsMapping(
((u.m, u.dimensionless_unscaled), ),
input_units_equivalencies={"x": equivalencies.spectral()})
result = model(Quantity(100, u.Hz))
assert result.unit == u.dimensionless_unscaled
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> from sunpy.database.tables import DatabaseEntry
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.search(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit')) # doctest: +REMOTE_DATA
>>> entry = DatabaseEntry._from_query_result_block(qr[0]) # doctest: +REMOTE_DATA
>>> entry.source # doctest: +REMOTE_DATA
SOHO
>>> entry.provider # doctest: +REMOTE_DATA
SDAC
>>> entry.physobs # doctest: +REMOTE_DATA
'intensity'
>>> entry.fileid # doctest: +REMOTE_DATA
/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.000042
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2001, 1, 1, 0, 0, 42), datetime.datetime(2001, 1, 1, 0, 0, 54))
>>> entry.instrument # doctest: +REMOTE_DATA
EIT
>>> entry.size # doctest: +REMOTE_DATA
2059.0
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(19.5, 19.5)
"""
time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start)
if not qr_block.time.end:
qr_block.time.end = qr_block.time.start
time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but Astropy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin),
equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax),
equivalencies.spectral())
source = getattr(qr_block, 'source', None)
provider = getattr(qr_block, 'provider', None)
fileid = getattr(qr_block, 'fileid', None)
instrument = getattr(qr_block, 'instrument', None)
size = getattr(qr_block, 'size', -1)
physobs = getattr(qr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
return cls(source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=size,
wavemin=wavemin,
wavemax=wavemax)
def freq_to_wave(freqcoord, waveframe):
return Wavelength(freqcoord.f.to(u.m, equivalencies=eq.spectral()))
def wave_to_freq(wavecoord, freqframe):
return Frequency(wavecoord.lam.to(u.Hz, equivalencies=eq.spectral()))
def to(self,
unit,
equivalencies=[],
doppler_rest=None,
doppler_convention=None):
"""
Return a new `~astropy.coordinates.SpectralQuantity` object with the specified unit.
By default, the ``spectral`` equivalency will be enabled, as well as
one of the Doppler equivalencies if converting to/from velocities.
Parameters
----------
unit : unit-like
An object that represents the unit to convert to. Must be
an `~astropy.units.UnitBase` object or a string parseable
by the `~astropy.units` package, and should be a spectral unit.
equivalencies : list of `~astropy.units.equivalencies.Equivalency`, optional
A list of equivalence pairs to try if the units are not
directly convertible (along with spectral).
See :ref:`astropy:unit_equivalencies`.
If not provided or ``[]``, spectral equivalencies will be used.
If `None`, no equivalencies will be applied at all, not even any
set globally or within a context.
doppler_rest : `~astropy.units.Quantity` ['speed'], optional
The rest value used when converting to/from velocities. This will
also be set at an attribute on the output
`~astropy.coordinates.SpectralQuantity`.
doppler_convention : {'relativistic', 'optical', 'radio'}, optional
The Doppler convention used when converting to/from velocities.
This will also be set at an attribute on the output
`~astropy.coordinates.SpectralQuantity`.
Returns
-------
`SpectralQuantity`
New spectral coordinate object with data converted to the new unit.
"""
# Make sure units can be passed as strings
unit = Unit(unit)
# If equivalencies is explicitly set to None, we should just use the
# default Quantity.to with equivalencies also set to None
if equivalencies is None:
result = super().to(unit, equivalencies=None)
result = result.view(self.__class__)
result.__array_finalize__(self)
return result
# FIXME: need to consider case where doppler equivalency is passed in
# equivalencies list, or is u.spectral equivalency is already passed
if doppler_rest is None:
doppler_rest = self._doppler_rest
if doppler_convention is None:
doppler_convention = self._doppler_convention
elif doppler_convention not in DOPPLER_CONVENTIONS:
raise ValueError(
f"doppler_convention should be one of {'/'.join(sorted(DOPPLER_CONVENTIONS))}"
)
if self.unit.is_equivalent(KMS) and unit.is_equivalent(KMS):
# Special case: if the current and final units are both velocity,
# and either the rest value or the convention are different, we
# need to convert back to frequency temporarily.
if doppler_convention is not None and self._doppler_convention is None:
raise ValueError("Original doppler_convention not set")
if doppler_rest is not None and self._doppler_rest is None:
raise ValueError("Original doppler_rest not set")
if doppler_rest is None and doppler_convention is None:
result = super().to(unit, equivalencies=equivalencies)
result = result.view(self.__class__)
result.__array_finalize__(self)
return result
elif (doppler_rest is None) is not (doppler_convention is None):
raise ValueError("Either both or neither doppler_rest and "
"doppler_convention should be defined for "
"velocity conversions")
vel_equiv1 = DOPPLER_CONVENTIONS[self._doppler_convention](
self._doppler_rest)
freq = super().to(si.Hz, equivalencies=equivalencies + vel_equiv1)
vel_equiv2 = DOPPLER_CONVENTIONS[doppler_convention](doppler_rest)
result = freq.to(unit, equivalencies=equivalencies + vel_equiv2)
else:
additional_equivalencies = eq.spectral()
if self.unit.is_equivalent(KMS) or unit.is_equivalent(KMS):
if doppler_convention is None:
raise ValueError(
"doppler_convention not set, cannot convert to/from velocities"
)
if doppler_rest is None:
raise ValueError(
"doppler_rest not set, cannot convert to/from velocities"
)
additional_equivalencies = additional_equivalencies + DOPPLER_CONVENTIONS[
doppler_convention](doppler_rest)
result = super().to(unit,
equivalencies=equivalencies +
additional_equivalencies)
# Since we have to explicitly specify when we want to keep this as a
# SpectralQuantity, we need to convert it back from a Quantity to
# a SpectralQuantity here. Note that we don't use __array_finalize__
# here since we might need to set the output doppler convention and
# rest based on the parameters passed to 'to'
result = result.view(self.__class__)
result.__array_finalize__(self)
result._doppler_convention = doppler_convention
result._doppler_rest = doppler_rest
return result
def wave_to_freq(wavecoord, freqframe):
return Frequency(wavecoord.lam.to(u.Hz, equivalencies=eq.spectral()))
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> qr = client.query(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit'))
>>> entry = DatabaseEntry.from_query_result_block(qr[0])
>>> entry.source
'SOHO'
>>> entry.provider
'SDAC'
>>> entry.physobs
'intensity'
>>> entry.fileid
'/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.010014'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2001, 1, 1, 1, 0, 14), datetime.datetime(2001, 1, 1, 1, 0, 21))
>>> entry.instrument
'EIT'
>>> entry.size
2059.0
>>> entry.wavemin, entry.wavemax
(17.1, 17.1)
"""
time_start = timestamp2datetime("%Y%m%d%H%M%S", qr_block.time.start)
time_end = timestamp2datetime("%Y%m%d%H%M%S", qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but AstroPy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin), equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax), equivalencies.spectral())
source = str(qr_block.source) if qr_block.source is not None else None
provider = str(qr_block.provider) if qr_block.provider is not None else None
fileid = str(qr_block.fileid) if qr_block.fileid is not None else None
instrument = str(qr_block.instrument) if qr_block.instrument is not None else None
physobs = getattr(qr_block, "physobs", None)
if physobs is not None:
physobs = str(physobs)
return cls(
source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=qr_block.size,
wavemin=wavemin,
wavemax=wavemax,
)
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None):
"""
Make a new :class:`DatabaseEntry` instance from a Fido search
result block.
Parameters
----------
sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock`
A query result block is usually not created directly; instead,
one gets instances of
``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating
over each element of a Fido search result.
default_waveunit : `str`, optional
The wavelength unit that is used if it cannot be found in the
`sr_block`.
"""
# All attributes of DatabaseEntry that are not in QueryResponseBlock
# are set as None for now.
source = getattr(sr_block, 'source', None)
provider = getattr(sr_block, 'provider', None)
physobs = getattr(sr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
instrument = getattr(sr_block, 'instrument', None)
time_start = sr_block.time.start
time_end = sr_block.time.end
wavelengths = getattr(sr_block, 'wave', None)
wavelength_temp = {}
if isinstance(wavelength_temp, tuple):
# Tuple of values
wavelength_temp['wavemin'] = wavelengths[0]
wavelength_temp['wavemax'] = wavelengths[1]
else:
# Single Value
wavelength_temp['wavemin'] = wavelength_temp['wavemax'] = wavelengths
final_values = {}
for key, val in wavelength_temp.items():
if isinstance(val, quantity.Quantity):
unit = getattr(val, 'unit', None)
if unit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
raise WaveunitNotFoundError(sr_block)
final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral())
elif val is None or np.isnan(val):
final_values[key] = val
wavemin = final_values['wavemin']
wavemax = final_values['wavemax']
# sr_block.url of a QueryResponseBlock attribute is stored in fileid
fileid = str(sr_block.url) if sr_block.url is not None else None
size = None
return cls(
source=source, provider=provider, physobs=physobs, fileid=fileid,
observation_time_start=time_start, observation_time_end=time_end,
instrument=instrument, size=size,
wavemin=wavemin, wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> from sunpy.database.tables import DatabaseEntry
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.search(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit')) # doctest: +REMOTE_DATA
>>> entry = DatabaseEntry._from_query_result_block(qr[0]) # doctest: +REMOTE_DATA
>>> entry.source # doctest: +REMOTE_DATA
SOHO
>>> entry.provider # doctest: +REMOTE_DATA
SDAC
>>> entry.physobs # doctest: +REMOTE_DATA
'intensity'
>>> entry.fileid # doctest: +REMOTE_DATA
/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.000042
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2001, 1, 1, 0, 0, 42), datetime.datetime(2001, 1, 1, 0, 0, 54))
>>> entry.instrument # doctest: +REMOTE_DATA
EIT
>>> entry.size # doctest: +REMOTE_DATA
2059.0
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(19.5, 19.5)
"""
time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start)
if not qr_block.time.end:
qr_block.time.end = qr_block.time.start
time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but Astropy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin),
equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax),
equivalencies.spectral())
source = getattr(qr_block, 'source', None)
provider = getattr(qr_block, 'provider', None)
fileid = getattr(qr_block, 'fileid', None)
instrument = getattr(qr_block, 'instrument', None)
size = getattr(qr_block, 'size', -1)
physobs = getattr(qr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
return cls(
source=source, provider=provider, physobs=physobs, fileid=fileid,
observation_time_start=time_start, observation_time_end=time_end,
instrument=instrument, size=size,
wavemin=wavemin, wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None):
"""Make a new :class:`DatabaseEntry` instance from a VSO query result
block. The values of :attr:`wavemin` and :attr:`wavemax` are converted
to nm (nanometres).
Parameters
----------
qr_block : suds.sudsobject.QueryResponseBlock
A query result block is usually not created directly; instead,
one gets instances of ``suds.sudsobject.QueryResponseBlock`` by
iterating over a VSO query result.
default_waveunit : str, optional
The wavelength unit that is used if it cannot be found in the
`qr_block`.
Examples
--------
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> qr = client.query(
... vso.attrs.Time('2001/1/1', '2001/1/2'),
... vso.attrs.Instrument('eit'))
>>> entry = DatabaseEntry.from_query_result_block(qr[0])
>>> entry.source
'SOHO'
>>> entry.provider
'SDAC'
>>> entry.physobs
'intensity'
>>> entry.fileid
'/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.010014'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2001, 1, 1, 1, 0, 14), datetime.datetime(2001, 1, 1, 1, 0, 21))
>>> entry.instrument
'EIT'
>>> entry.size
2059.0
>>> entry.wavemin, entry.wavemax
(17.1, 17.1)
"""
time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start)
time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end)
wave = qr_block.wave
unit = None
if wave.waveunit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
# some query response blocks store the unit "kev",
# but AstroPy only understands "keV". See issue #766.
waveunit = wave.waveunit
if waveunit == "kev":
waveunit = "keV"
unit = Unit(waveunit)
if wave.wavemin is None:
wavemin = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemin = unit.to(nm, float(wave.wavemin),
equivalencies.spectral())
if wave.wavemax is None:
wavemax = None
else:
if unit is None:
raise WaveunitNotFoundError(qr_block)
wavemax = unit.to(nm, float(wave.wavemax),
equivalencies.spectral())
source = str(qr_block.source) if qr_block.source is not None else None
provider = str(
qr_block.provider) if qr_block.provider is not None else None
fileid = str(qr_block.fileid) if qr_block.fileid is not None else None
instrument = str(
qr_block.instrument) if qr_block.instrument is not None else None
physobs = getattr(qr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
return cls(source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=qr_block.size,
wavemin=wavemin,
wavemax=wavemax)
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in root.attrs.iteritems():
typ = key[0]
if typ == 'tag':
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == 'fitsheaderentry':
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(
not_(
and_(
DatabaseEntry.fits_header_entries.any(
key_criterion),
DatabaseEntry.fits_header_entries.any(
value_criterion))))
else:
query = query.filter(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(
value_criterion)))
elif typ == 'download time':
start, end, inverted = value
if inverted:
query = query.filter(
~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(
DatabaseEntry.download_time.between(start, end))
elif typ == 'path':
path, inverted = value
if inverted:
query = query.filter(
or_(DatabaseEntry.path != path,
DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == 'wave':
min_, max_, unit = value
waveunit = Unit(unit)
# convert min_ and max_ to nm from the unit `waveunit`
wavemin = waveunit.to(nm, min_, equivalencies.spectral())
wavemax = waveunit.to(nm, max_, equivalencies.spectral())
query = query.filter(
and_(DatabaseEntry.wavemin >= wavemin,
DatabaseEntry.wavemax <= wavemax))
elif typ == 'time':
start, end, near = value
query = query.filter(
and_(DatabaseEntry.observation_time_start < end,
DatabaseEntry.observation_time_end > start))
else:
if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(
SUPPORTED_NONVSO_ATTRS):
raise NotImplementedError(
"The attribute {0!r} is not yet supported to query a database."
.format(typ))
query = query.filter_by(**{typ: value})
return query.all()
def freq_to_wave(freqcoord, waveframe):
return Wavelength(freqcoord.f.to(u.m, equivalencies=eq.spectral()))
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None):
"""
Make a new :class:`DatabaseEntry` instance from a Fido search
result block.
Parameters
----------
sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock`
A query result block is usually not created directly; instead,
one gets instances of
``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating
over each element of a Fido search result.
default_waveunit : `str`, optional
The wavelength unit that is used if it cannot be found in the
`sr_block`.
"""
# All attributes of DatabaseEntry that are not in QueryResponseBlock
# are set as None for now.
source = getattr(sr_block, 'source', None)
provider = getattr(sr_block, 'provider', None)
physobs = getattr(sr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
instrument = getattr(sr_block, 'instrument', None)
time_start = sr_block.time.start
time_end = sr_block.time.end
wavelengths = getattr(sr_block, 'wave', None)
wavelength_temp = {}
if isinstance(wavelength_temp, tuple):
# Tuple of values
wavelength_temp['wavemin'] = wavelengths[0]
wavelength_temp['wavemax'] = wavelengths[1]
else:
# Single Value
wavelength_temp['wavemin'] = wavelength_temp[
'wavemax'] = wavelengths
final_values = {}
for key, val in wavelength_temp.items():
if isinstance(val, quantity.Quantity):
unit = getattr(val, 'unit', None)
if unit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
raise WaveunitNotFoundError(sr_block)
final_values[key] = unit.to(nm, float(val.value),
equivalencies.spectral())
elif val is None or np.isnan(val):
final_values[key] = val
wavemin = final_values['wavemin']
wavemax = final_values['wavemax']
# sr_block.url of a QueryResponseBlock attribute is stored in fileid
fileid = str(sr_block.url) if sr_block.url is not None else None
size = None
return cls(source=source,
provider=provider,
physobs=physobs,
fileid=fileid,
observation_time_start=time_start,
observation_time_end=time_end,
instrument=instrument,
size=size,
wavemin=wavemin,
wavemax=wavemax)
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None):
"""
Make a new :class:`DatabaseEntry` instance from a Fido search
result block.
Parameters
----------
sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock`
A query result block is usually not created directly; instead,
one gets instances of
``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating
over each element of a Fido search result.
default_waveunit : `str`, optional
The wavelength unit that is used if it cannot be found in the
`sr_block`.
Examples
--------
>>> from sunpy.net import Fido, attrs
>>> from sunpy.database.tables import DatabaseEntry
>>> sr = Fido.search(attrs.Time("2012/1/1", "2012/1/2"),
... attrs.Instrument('lyra'))
>>> entry = DatabaseEntry._from_fido_search_result_block(sr[0][0])
>>> entry.source
'Proba2'
>>> entry.provider
'esa'
>>> entry.physobs
'irradiance'
>>> entry.fileid
'http://proba2.oma.be/lyra/data/bsd/2012/01/01/lyra_20120101-000000_lev2_std.fits'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2012, 1, 1, 0, 0), datetime.datetime(2012, 1, 2, 0, 0))
>>> entry.instrument
'lyra'
"""
# All attributes of DatabaseEntry that are not in QueryResponseBlock
# are set as None for now.
source = getattr(sr_block, 'source', None)
provider = getattr(sr_block, 'provider', None)
physobs = getattr(sr_block, 'physobs', None)
if physobs is not None:
physobs = str(physobs)
instrument = getattr(sr_block, 'instrument', None)
time_start = sr_block.time.start
time_end = sr_block.time.end
wavelengths = getattr(sr_block, 'wave', None)
wavelength_temp = {}
if isinstance(wavelength_temp, tuple):
# Tuple of values
wavelength_temp['wavemin'] = wavelengths[0]
wavelength_temp['wavemax'] = wavelengths[1]
else:
# Single Value
wavelength_temp['wavemin'] = wavelength_temp['wavemax'] = wavelengths
final_values = {}
for key, val in wavelength_temp.items():
if isinstance(val, quantity.Quantity):
unit = getattr(val, 'unit', None)
if unit is None:
if default_waveunit is not None:
unit = Unit(default_waveunit)
else:
raise WaveunitNotFoundError(sr_block)
final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral())
elif val is None or np.isnan(val):
final_values[key] = val
wavemin = final_values['wavemin']
wavemax = final_values['wavemax']
# sr_block.url of a QueryResponseBlock attribute is stored in fileid
fileid = str(sr_block.url) if sr_block.url is not None else None
size = None
return cls(
source=source, provider=provider, physobs=physobs, fileid=fileid,
observation_time_start=time_start, observation_time_end=time_end,
instrument=instrument, size=size,
wavemin=wavemin, wavemax=wavemax)
def entries_from_file(file,
default_waveunit=None,
time_string_parse_format=None):
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
time_string_parse_format : str, optional
Fallback timestamp format which will be passed to
`~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to
automatically read the `date-obs` metadata.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> from sunpy.database.tables import entries_from_file
>>> import sunpy.data.sample # doctest: +REMOTE_DATA
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA
>>> len(entries) # doctest: +REMOTE_DATA
1
>>> entry = entries.pop() # doctest: +REMOTE_DATA
>>> entry.instrument # doctest: +REMOTE_DATA
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None)
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA
111
"""
headers = fits.get_header(file)
if isinstance(file, (str, six.text_type)):
filename = file
else:
filename = getattr(file, 'name', None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in six.iteritems(header):
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == '':
value = str(value)
elif key == 'KEYCOMMENTS':
for k, v in six.iteritems(value):
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
entry.hdu_index = headers.index(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == 'INSTRUME':
entry.instrument = value
elif key == 'WAVELNTH':
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(
nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ('DATE-END', 'DATE_END'):
entry.observation_time_end = parse_time(
value, _time_string_parse_format=time_string_parse_format)
elif key in ('DATE-OBS', 'DATE_OBS'):
entry.observation_time_start = parse_time(
value, _time_string_parse_format=time_string_parse_format)
yield entry
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in root.attrs.iteritems():
typ = key[0]
if typ == "tag":
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == "fitsheaderentry":
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(
not_(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion),
)
)
)
else:
query = query.filter(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion),
)
)
elif typ == "download time":
start, end, inverted = value
if inverted:
query = query.filter(~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(DatabaseEntry.download_time.between(start, end))
elif typ == "path":
path, inverted = value
if inverted:
query = query.filter(or_(DatabaseEntry.path != path, DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == "wave":
min_, max_, unit = value
waveunit = Unit(unit)
# convert min_ and max_ to nm from the unit `waveunit`
wavemin = waveunit.to(nm, min_, equivalencies.spectral())
wavemax = waveunit.to(nm, max_, equivalencies.spectral())
query = query.filter(and_(DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax))
elif typ == "time":
start, end, near = value
query = query.filter(
and_(DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start)
)
else:
if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(SUPPORTED_NONVSO_ATTRS):
raise NotImplementedError("The attribute {0!r} is not yet supported to query a database.".format(typ))
query = query.filter_by(**{typ: value})
return query.all()
def entries_from_file(file, default_waveunit=None):
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE))
>>> len(entries)
1
>>> entry = entries.pop()
>>> entry.instrument
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2012, 1, 1, 0, 16, 7, 836000), None)
>>> entry.wavemin, entry.wavemax
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries)
112
"""
headers = fits.get_header(file)
if isinstance(file, (str, unicode)):
filename = file
else:
filename = getattr(file, "name", None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in header.iteritems():
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == "":
value = str(value)
elif key == "KEYCOMMENTS":
for k, v in value.iteritems():
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == "INSTRUME":
entry.instrument = value
elif key == "WAVELNTH":
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ("DATE-END", "DATE_END"):
entry.observation_time_end = parse_time(value)
elif key in ("DATE-OBS", "DATE_OBS"):
entry.observation_time_start = parse_time(value)
yield entry
def entries_from_file(file, default_waveunit=None,
time_string_parse_format=''):
# Note: time_string_parse_format='' so that None won't be passed to Time.strptime
# (which would make strptime freak out, if I remember correctly).
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
time_string_parse_format : str, optional
Fallback timestamp format which will be passed to
`~astropy.time.Time.strptime` if `sunpy.time.parse_time` is unable to
automatically read the `date-obs` metadata.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> from sunpy.database.tables import entries_from_file
>>> import sunpy.data.sample # doctest: +REMOTE_DATA
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA
>>> len(entries) # doctest: +REMOTE_DATA
1
>>> entry = entries.pop() # doctest: +REMOTE_DATA
>>> entry.instrument # doctest: +REMOTE_DATA
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None)
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA
111
"""
headers = fits.get_header(file)
# This just checks for blank default headers
# due to compression.
for header in headers:
if header == DEFAULT_HEADER:
headers.remove(header)
if isinstance(file, str):
filename = file
else:
filename = getattr(file, 'name', None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in header.items():
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == '':
value = str(value)
elif key == 'KEYCOMMENTS':
for k, v in value.items():
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
entry.hdu_index = headers.index(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == 'INSTRUME':
entry.instrument = value
elif key == 'WAVELNTH':
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(
nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ('DATE-END', 'DATE_END'):
try:
dt = parse_time(value).datetime
except ValueError:
dt = Time.strptime(value, time_string_parse_format).datetime
entry.observation_time_end = dt
elif key in ('DATE-OBS', 'DATE_OBS'):
try:
dt = parse_time(value).datetime
except ValueError:
dt = Time.strptime(value, time_string_parse_format).datetime
entry.observation_time_start = dt
yield entry
