def get_equivalencies(): """ Return a list of example equivalencies for testing serialization. """ return [ eq.plate_scale(.3 * u.deg / u.mm), eq.pixel_scale(.5 * u.deg / u.pix), eq.spectral_density(350 * u.nm, factor=2), eq.spectral_density(350 * u.nm), eq.spectral(), eq.brightness_temperature(500 * u.GHz), eq.brightness_temperature(500 * u.GHz, beam_area=23 * u.sr), eq.with_H0(), eq.temperature_energy(), eq.temperature(), eq.thermodynamic_temperature(300 * u.Hz), eq.thermodynamic_temperature(140 * u.GHz, Planck15.Tcmb0), eq.beam_angular_area(3 * u.sr), eq.mass_energy(), eq.molar_mass_amu(), eq.doppler_relativistic(2 * u.m), eq.doppler_optical(2 * u.nm), eq.doppler_radio(2 * u.Hz), eq.parallax(), eq.logarithmic(), eq.dimensionless_angles(), eq.spectral() + eq.temperature(), (eq.spectral_density(35 * u.nm) + eq.brightness_temperature(5 * u.Hz, beam_area=2 * u.sr)), (eq.spectral() + eq.spectral_density(35 * u.nm) + eq.brightness_temperature(5 * u.Hz, beam_area=2 * u.sr)) ]
def _create(wlk, root, session): query = session.query(DatabaseEntry) for key, value in root.attrs.iteritems(): typ = key[0] if typ == 'tag': criterion = TableTag.name.in_([value]) # `key[1]` is here the `inverted` attribute of the tag. That means # that if it is True, the given tag must not be included in the # resulting entries. if key[1]: query = query.filter(~DatabaseEntry.tags.any(criterion)) else: query = query.filter(DatabaseEntry.tags.any(criterion)) elif typ == 'fitsheaderentry': key, val, inverted = value key_criterion = TableFitsHeaderEntry.key == key value_criterion = TableFitsHeaderEntry.value == val if inverted: query = query.filter(not_(and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any(value_criterion)))) else: query = query.filter(and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any(value_criterion))) elif typ == 'download time': start, end, inverted = value if inverted: query = query.filter( ~DatabaseEntry.download_time.between(start, end)) else: query = query.filter( DatabaseEntry.download_time.between(start, end)) elif typ == 'path': path, inverted = value if inverted: query = query.filter(or_( DatabaseEntry.path != path, DatabaseEntry.path == None)) else: query = query.filter(DatabaseEntry.path == path) elif typ == 'wave': min_, max_, unit = value waveunit = Unit(unit) # convert min_ and max_ to nm from the unit `waveunit` wavemin = waveunit.to(nm, min_, equivalencies.spectral()) wavemax = waveunit.to(nm, max_, equivalencies.spectral()) query = query.filter(and_( DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax)) elif typ == 'time': start, end, near = value query = query.filter(and_( DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start)) else: query = query.filter_by(**{typ: value}) return query.all()
def test_equivalencies(): model = UnitsMapping(((u.m, u.dimensionless_unscaled), )) with pytest.raises(UnitsError): model(Quantity(100, u.Hz)) model = UnitsMapping( ((u.m, u.dimensionless_unscaled), ), input_units_equivalencies={"x": equivalencies.spectral()}) result = model(Quantity(100, u.Hz)) assert result.unit == u.dimensionless_unscaled
def _from_query_result_block(cls, qr_block, default_waveunit=None): """Make a new :class:`DatabaseEntry` instance from a VSO query result block. The values of :attr:`wavemin` and :attr:`wavemax` are converted to nm (nanometres). Parameters ---------- qr_block : suds.sudsobject.QueryResponseBlock A query result block is usually not created directly; instead, one gets instances of ``suds.sudsobject.QueryResponseBlock`` by iterating over a VSO query result. default_waveunit : str, optional The wavelength unit that is used if it cannot be found in the `qr_block`. Examples -------- >>> from sunpy.net import vso >>> from sunpy.database.tables import DatabaseEntry >>> client = vso.VSOClient() # doctest: +REMOTE_DATA >>> qr = client.search( ... vso.attrs.Time('2001/1/1', '2001/1/2'), ... vso.attrs.Instrument('eit')) # doctest: +REMOTE_DATA >>> entry = DatabaseEntry._from_query_result_block(qr[0]) # doctest: +REMOTE_DATA >>> entry.source # doctest: +REMOTE_DATA SOHO >>> entry.provider # doctest: +REMOTE_DATA SDAC >>> entry.physobs # doctest: +REMOTE_DATA 'intensity' >>> entry.fileid # doctest: +REMOTE_DATA /archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.000042 >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2001, 1, 1, 0, 0, 42), datetime.datetime(2001, 1, 1, 0, 0, 54)) >>> entry.instrument # doctest: +REMOTE_DATA EIT >>> entry.size # doctest: +REMOTE_DATA 2059.0 >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (19.5, 19.5) """ time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start) if not qr_block.time.end: qr_block.time.end = qr_block.time.start time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end) wave = qr_block.wave unit = None if wave.waveunit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: # some query response blocks store the unit "kev", # but Astropy only understands "keV". See issue #766. waveunit = wave.waveunit if waveunit == "kev": waveunit = "keV" unit = Unit(waveunit) if wave.wavemin is None: wavemin = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemin = unit.to(nm, float(wave.wavemin), equivalencies.spectral()) if wave.wavemax is None: wavemax = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemax = unit.to(nm, float(wave.wavemax), equivalencies.spectral()) source = getattr(qr_block, 'source', None) provider = getattr(qr_block, 'provider', None) fileid = getattr(qr_block, 'fileid', None) instrument = getattr(qr_block, 'instrument', None) size = getattr(qr_block, 'size', -1) physobs = getattr(qr_block, 'physobs', None) if physobs is not None: physobs = str(physobs) return cls(source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=size, wavemin=wavemin, wavemax=wavemax)
def freq_to_wave(freqcoord, waveframe): return Wavelength(freqcoord.f.to(u.m, equivalencies=eq.spectral()))
def wave_to_freq(wavecoord, freqframe): return Frequency(wavecoord.lam.to(u.Hz, equivalencies=eq.spectral()))
def to(self, unit, equivalencies=[], doppler_rest=None, doppler_convention=None): """ Return a new `~astropy.coordinates.SpectralQuantity` object with the specified unit. By default, the ``spectral`` equivalency will be enabled, as well as one of the Doppler equivalencies if converting to/from velocities. Parameters ---------- unit : unit-like An object that represents the unit to convert to. Must be an `~astropy.units.UnitBase` object or a string parseable by the `~astropy.units` package, and should be a spectral unit. equivalencies : list of `~astropy.units.equivalencies.Equivalency`, optional A list of equivalence pairs to try if the units are not directly convertible (along with spectral). See :ref:`astropy:unit_equivalencies`. If not provided or ``[]``, spectral equivalencies will be used. If `None`, no equivalencies will be applied at all, not even any set globally or within a context. doppler_rest : `~astropy.units.Quantity` ['speed'], optional The rest value used when converting to/from velocities. This will also be set at an attribute on the output `~astropy.coordinates.SpectralQuantity`. doppler_convention : {'relativistic', 'optical', 'radio'}, optional The Doppler convention used when converting to/from velocities. This will also be set at an attribute on the output `~astropy.coordinates.SpectralQuantity`. Returns ------- `SpectralQuantity` New spectral coordinate object with data converted to the new unit. """ # Make sure units can be passed as strings unit = Unit(unit) # If equivalencies is explicitly set to None, we should just use the # default Quantity.to with equivalencies also set to None if equivalencies is None: result = super().to(unit, equivalencies=None) result = result.view(self.__class__) result.__array_finalize__(self) return result # FIXME: need to consider case where doppler equivalency is passed in # equivalencies list, or is u.spectral equivalency is already passed if doppler_rest is None: doppler_rest = self._doppler_rest if doppler_convention is None: doppler_convention = self._doppler_convention elif doppler_convention not in DOPPLER_CONVENTIONS: raise ValueError( f"doppler_convention should be one of {'/'.join(sorted(DOPPLER_CONVENTIONS))}" ) if self.unit.is_equivalent(KMS) and unit.is_equivalent(KMS): # Special case: if the current and final units are both velocity, # and either the rest value or the convention are different, we # need to convert back to frequency temporarily. if doppler_convention is not None and self._doppler_convention is None: raise ValueError("Original doppler_convention not set") if doppler_rest is not None and self._doppler_rest is None: raise ValueError("Original doppler_rest not set") if doppler_rest is None and doppler_convention is None: result = super().to(unit, equivalencies=equivalencies) result = result.view(self.__class__) result.__array_finalize__(self) return result elif (doppler_rest is None) is not (doppler_convention is None): raise ValueError("Either both or neither doppler_rest and " "doppler_convention should be defined for " "velocity conversions") vel_equiv1 = DOPPLER_CONVENTIONS[self._doppler_convention]( self._doppler_rest) freq = super().to(si.Hz, equivalencies=equivalencies + vel_equiv1) vel_equiv2 = DOPPLER_CONVENTIONS[doppler_convention](doppler_rest) result = freq.to(unit, equivalencies=equivalencies + vel_equiv2) else: additional_equivalencies = eq.spectral() if self.unit.is_equivalent(KMS) or unit.is_equivalent(KMS): if doppler_convention is None: raise ValueError( "doppler_convention not set, cannot convert to/from velocities" ) if doppler_rest is None: raise ValueError( "doppler_rest not set, cannot convert to/from velocities" ) additional_equivalencies = additional_equivalencies + DOPPLER_CONVENTIONS[ doppler_convention](doppler_rest) result = super().to(unit, equivalencies=equivalencies + additional_equivalencies) # Since we have to explicitly specify when we want to keep this as a # SpectralQuantity, we need to convert it back from a Quantity to # a SpectralQuantity here. Note that we don't use __array_finalize__ # here since we might need to set the output doppler convention and # rest based on the parameters passed to 'to' result = result.view(self.__class__) result.__array_finalize__(self) result._doppler_convention = doppler_convention result._doppler_rest = doppler_rest return result
def _from_query_result_block(cls, qr_block, default_waveunit=None): """Make a new :class:`DatabaseEntry` instance from a VSO query result block. The values of :attr:`wavemin` and :attr:`wavemax` are converted to nm (nanometres). Parameters ---------- qr_block : suds.sudsobject.QueryResponseBlock A query result block is usually not created directly; instead, one gets instances of ``suds.sudsobject.QueryResponseBlock`` by iterating over a VSO query result. default_waveunit : str, optional The wavelength unit that is used if it cannot be found in the `qr_block`. Examples -------- >>> from sunpy.net import vso >>> client = vso.VSOClient() >>> qr = client.query( ... vso.attrs.Time('2001/1/1', '2001/1/2'), ... vso.attrs.Instrument('eit')) >>> entry = DatabaseEntry.from_query_result_block(qr[0]) >>> entry.source 'SOHO' >>> entry.provider 'SDAC' >>> entry.physobs 'intensity' >>> entry.fileid '/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.010014' >>> entry.observation_time_start, entry.observation_time_end (datetime.datetime(2001, 1, 1, 1, 0, 14), datetime.datetime(2001, 1, 1, 1, 0, 21)) >>> entry.instrument 'EIT' >>> entry.size 2059.0 >>> entry.wavemin, entry.wavemax (17.1, 17.1) """ time_start = timestamp2datetime("%Y%m%d%H%M%S", qr_block.time.start) time_end = timestamp2datetime("%Y%m%d%H%M%S", qr_block.time.end) wave = qr_block.wave unit = None if wave.waveunit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: # some query response blocks store the unit "kev", # but AstroPy only understands "keV". See issue #766. waveunit = wave.waveunit if waveunit == "kev": waveunit = "keV" unit = Unit(waveunit) if wave.wavemin is None: wavemin = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemin = unit.to(nm, float(wave.wavemin), equivalencies.spectral()) if wave.wavemax is None: wavemax = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemax = unit.to(nm, float(wave.wavemax), equivalencies.spectral()) source = str(qr_block.source) if qr_block.source is not None else None provider = str(qr_block.provider) if qr_block.provider is not None else None fileid = str(qr_block.fileid) if qr_block.fileid is not None else None instrument = str(qr_block.instrument) if qr_block.instrument is not None else None physobs = getattr(qr_block, "physobs", None) if physobs is not None: physobs = str(physobs) return cls( source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=qr_block.size, wavemin=wavemin, wavemax=wavemax, )
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None): """ Make a new :class:`DatabaseEntry` instance from a Fido search result block. Parameters ---------- sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock` A query result block is usually not created directly; instead, one gets instances of ``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating over each element of a Fido search result. default_waveunit : `str`, optional The wavelength unit that is used if it cannot be found in the `sr_block`. """ # All attributes of DatabaseEntry that are not in QueryResponseBlock # are set as None for now. source = getattr(sr_block, 'source', None) provider = getattr(sr_block, 'provider', None) physobs = getattr(sr_block, 'physobs', None) if physobs is not None: physobs = str(physobs) instrument = getattr(sr_block, 'instrument', None) time_start = sr_block.time.start time_end = sr_block.time.end wavelengths = getattr(sr_block, 'wave', None) wavelength_temp = {} if isinstance(wavelength_temp, tuple): # Tuple of values wavelength_temp['wavemin'] = wavelengths[0] wavelength_temp['wavemax'] = wavelengths[1] else: # Single Value wavelength_temp['wavemin'] = wavelength_temp['wavemax'] = wavelengths final_values = {} for key, val in wavelength_temp.items(): if isinstance(val, quantity.Quantity): unit = getattr(val, 'unit', None) if unit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: raise WaveunitNotFoundError(sr_block) final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral()) elif val is None or np.isnan(val): final_values[key] = val wavemin = final_values['wavemin'] wavemax = final_values['wavemax'] # sr_block.url of a QueryResponseBlock attribute is stored in fileid fileid = str(sr_block.url) if sr_block.url is not None else None size = None return cls( source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=size, wavemin=wavemin, wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None): """Make a new :class:`DatabaseEntry` instance from a VSO query result block. The values of :attr:`wavemin` and :attr:`wavemax` are converted to nm (nanometres). Parameters ---------- qr_block : suds.sudsobject.QueryResponseBlock A query result block is usually not created directly; instead, one gets instances of ``suds.sudsobject.QueryResponseBlock`` by iterating over a VSO query result. default_waveunit : str, optional The wavelength unit that is used if it cannot be found in the `qr_block`. Examples -------- >>> from sunpy.net import vso >>> from sunpy.database.tables import DatabaseEntry >>> client = vso.VSOClient() # doctest: +REMOTE_DATA >>> qr = client.search( ... vso.attrs.Time('2001/1/1', '2001/1/2'), ... vso.attrs.Instrument('eit')) # doctest: +REMOTE_DATA >>> entry = DatabaseEntry._from_query_result_block(qr[0]) # doctest: +REMOTE_DATA >>> entry.source # doctest: +REMOTE_DATA SOHO >>> entry.provider # doctest: +REMOTE_DATA SDAC >>> entry.physobs # doctest: +REMOTE_DATA 'intensity' >>> entry.fileid # doctest: +REMOTE_DATA /archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.000042 >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2001, 1, 1, 0, 0, 42), datetime.datetime(2001, 1, 1, 0, 0, 54)) >>> entry.instrument # doctest: +REMOTE_DATA EIT >>> entry.size # doctest: +REMOTE_DATA 2059.0 >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (19.5, 19.5) """ time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start) if not qr_block.time.end: qr_block.time.end = qr_block.time.start time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end) wave = qr_block.wave unit = None if wave.waveunit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: # some query response blocks store the unit "kev", # but Astropy only understands "keV". See issue #766. waveunit = wave.waveunit if waveunit == "kev": waveunit = "keV" unit = Unit(waveunit) if wave.wavemin is None: wavemin = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemin = unit.to(nm, float(wave.wavemin), equivalencies.spectral()) if wave.wavemax is None: wavemax = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemax = unit.to(nm, float(wave.wavemax), equivalencies.spectral()) source = getattr(qr_block, 'source', None) provider = getattr(qr_block, 'provider', None) fileid = getattr(qr_block, 'fileid', None) instrument = getattr(qr_block, 'instrument', None) size = getattr(qr_block, 'size', -1) physobs = getattr(qr_block, 'physobs', None) if physobs is not None: physobs = str(physobs) return cls( source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=size, wavemin=wavemin, wavemax=wavemax)
def _from_query_result_block(cls, qr_block, default_waveunit=None): """Make a new :class:`DatabaseEntry` instance from a VSO query result block. The values of :attr:`wavemin` and :attr:`wavemax` are converted to nm (nanometres). Parameters ---------- qr_block : suds.sudsobject.QueryResponseBlock A query result block is usually not created directly; instead, one gets instances of ``suds.sudsobject.QueryResponseBlock`` by iterating over a VSO query result. default_waveunit : str, optional The wavelength unit that is used if it cannot be found in the `qr_block`. Examples -------- >>> from sunpy.net import vso >>> client = vso.VSOClient() >>> qr = client.query( ... vso.attrs.Time('2001/1/1', '2001/1/2'), ... vso.attrs.Instrument('eit')) >>> entry = DatabaseEntry.from_query_result_block(qr[0]) >>> entry.source 'SOHO' >>> entry.provider 'SDAC' >>> entry.physobs 'intensity' >>> entry.fileid '/archive/soho/private/data/processed/eit/lz/2001/01/efz20010101.010014' >>> entry.observation_time_start, entry.observation_time_end (datetime.datetime(2001, 1, 1, 1, 0, 14), datetime.datetime(2001, 1, 1, 1, 0, 21)) >>> entry.instrument 'EIT' >>> entry.size 2059.0 >>> entry.wavemin, entry.wavemax (17.1, 17.1) """ time_start = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.start) time_end = timestamp2datetime('%Y%m%d%H%M%S', qr_block.time.end) wave = qr_block.wave unit = None if wave.waveunit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: # some query response blocks store the unit "kev", # but AstroPy only understands "keV". See issue #766. waveunit = wave.waveunit if waveunit == "kev": waveunit = "keV" unit = Unit(waveunit) if wave.wavemin is None: wavemin = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemin = unit.to(nm, float(wave.wavemin), equivalencies.spectral()) if wave.wavemax is None: wavemax = None else: if unit is None: raise WaveunitNotFoundError(qr_block) wavemax = unit.to(nm, float(wave.wavemax), equivalencies.spectral()) source = str(qr_block.source) if qr_block.source is not None else None provider = str( qr_block.provider) if qr_block.provider is not None else None fileid = str(qr_block.fileid) if qr_block.fileid is not None else None instrument = str( qr_block.instrument) if qr_block.instrument is not None else None physobs = getattr(qr_block, 'physobs', None) if physobs is not None: physobs = str(physobs) return cls(source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=qr_block.size, wavemin=wavemin, wavemax=wavemax)
def _create(wlk, root, session): query = session.query(DatabaseEntry) for key, value in root.attrs.iteritems(): typ = key[0] if typ == 'tag': criterion = TableTag.name.in_([value]) # `key[1]` is here the `inverted` attribute of the tag. That means # that if it is True, the given tag must not be included in the # resulting entries. if key[1]: query = query.filter(~DatabaseEntry.tags.any(criterion)) else: query = query.filter(DatabaseEntry.tags.any(criterion)) elif typ == 'fitsheaderentry': key, val, inverted = value key_criterion = TableFitsHeaderEntry.key == key value_criterion = TableFitsHeaderEntry.value == val if inverted: query = query.filter( not_( and_( DatabaseEntry.fits_header_entries.any( key_criterion), DatabaseEntry.fits_header_entries.any( value_criterion)))) else: query = query.filter( and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any( value_criterion))) elif typ == 'download time': start, end, inverted = value if inverted: query = query.filter( ~DatabaseEntry.download_time.between(start, end)) else: query = query.filter( DatabaseEntry.download_time.between(start, end)) elif typ == 'path': path, inverted = value if inverted: query = query.filter( or_(DatabaseEntry.path != path, DatabaseEntry.path == None)) else: query = query.filter(DatabaseEntry.path == path) elif typ == 'wave': min_, max_, unit = value waveunit = Unit(unit) # convert min_ and max_ to nm from the unit `waveunit` wavemin = waveunit.to(nm, min_, equivalencies.spectral()) wavemax = waveunit.to(nm, max_, equivalencies.spectral()) query = query.filter( and_(DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax)) elif typ == 'time': start, end, near = value query = query.filter( and_(DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start)) else: if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union( SUPPORTED_NONVSO_ATTRS): raise NotImplementedError( "The attribute {0!r} is not yet supported to query a database." .format(typ)) query = query.filter_by(**{typ: value}) return query.all()
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None): """ Make a new :class:`DatabaseEntry` instance from a Fido search result block. Parameters ---------- sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock` A query result block is usually not created directly; instead, one gets instances of ``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating over each element of a Fido search result. default_waveunit : `str`, optional The wavelength unit that is used if it cannot be found in the `sr_block`. """ # All attributes of DatabaseEntry that are not in QueryResponseBlock # are set as None for now. source = getattr(sr_block, 'source', None) provider = getattr(sr_block, 'provider', None) physobs = getattr(sr_block, 'physobs', None) if physobs is not None: physobs = str(physobs) instrument = getattr(sr_block, 'instrument', None) time_start = sr_block.time.start time_end = sr_block.time.end wavelengths = getattr(sr_block, 'wave', None) wavelength_temp = {} if isinstance(wavelength_temp, tuple): # Tuple of values wavelength_temp['wavemin'] = wavelengths[0] wavelength_temp['wavemax'] = wavelengths[1] else: # Single Value wavelength_temp['wavemin'] = wavelength_temp[ 'wavemax'] = wavelengths final_values = {} for key, val in wavelength_temp.items(): if isinstance(val, quantity.Quantity): unit = getattr(val, 'unit', None) if unit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: raise WaveunitNotFoundError(sr_block) final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral()) elif val is None or np.isnan(val): final_values[key] = val wavemin = final_values['wavemin'] wavemax = final_values['wavemax'] # sr_block.url of a QueryResponseBlock attribute is stored in fileid fileid = str(sr_block.url) if sr_block.url is not None else None size = None return cls(source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=size, wavemin=wavemin, wavemax=wavemax)
def _from_fido_search_result_block(cls, sr_block, default_waveunit=None): """ Make a new :class:`DatabaseEntry` instance from a Fido search result block. Parameters ---------- sr_block : `sunpy.net.dataretriever.client.QueryResponseBlock` A query result block is usually not created directly; instead, one gets instances of ``sunpy.net.dataretriever.client.QueryResponseBlock`` by iterating over each element of a Fido search result. default_waveunit : `str`, optional The wavelength unit that is used if it cannot be found in the `sr_block`. Examples -------- >>> from sunpy.net import Fido, attrs >>> from sunpy.database.tables import DatabaseEntry >>> sr = Fido.search(attrs.Time("2012/1/1", "2012/1/2"), ... attrs.Instrument('lyra')) >>> entry = DatabaseEntry._from_fido_search_result_block(sr[0][0]) >>> entry.source 'Proba2' >>> entry.provider 'esa' >>> entry.physobs 'irradiance' >>> entry.fileid 'http://proba2.oma.be/lyra/data/bsd/2012/01/01/lyra_20120101-000000_lev2_std.fits' >>> entry.observation_time_start, entry.observation_time_end (datetime.datetime(2012, 1, 1, 0, 0), datetime.datetime(2012, 1, 2, 0, 0)) >>> entry.instrument 'lyra' """ # All attributes of DatabaseEntry that are not in QueryResponseBlock # are set as None for now. source = getattr(sr_block, 'source', None) provider = getattr(sr_block, 'provider', None) physobs = getattr(sr_block, 'physobs', None) if physobs is not None: physobs = str(physobs) instrument = getattr(sr_block, 'instrument', None) time_start = sr_block.time.start time_end = sr_block.time.end wavelengths = getattr(sr_block, 'wave', None) wavelength_temp = {} if isinstance(wavelength_temp, tuple): # Tuple of values wavelength_temp['wavemin'] = wavelengths[0] wavelength_temp['wavemax'] = wavelengths[1] else: # Single Value wavelength_temp['wavemin'] = wavelength_temp['wavemax'] = wavelengths final_values = {} for key, val in wavelength_temp.items(): if isinstance(val, quantity.Quantity): unit = getattr(val, 'unit', None) if unit is None: if default_waveunit is not None: unit = Unit(default_waveunit) else: raise WaveunitNotFoundError(sr_block) final_values[key] = unit.to(nm, float(val.value), equivalencies.spectral()) elif val is None or np.isnan(val): final_values[key] = val wavemin = final_values['wavemin'] wavemax = final_values['wavemax'] # sr_block.url of a QueryResponseBlock attribute is stored in fileid fileid = str(sr_block.url) if sr_block.url is not None else None size = None return cls( source=source, provider=provider, physobs=physobs, fileid=fileid, observation_time_start=time_start, observation_time_end=time_end, instrument=instrument, size=size, wavemin=wavemin, wavemax=wavemax)
def entries_from_file(file, default_waveunit=None, time_string_parse_format=None): """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. time_string_parse_format : str, optional Fallback timestamp format which will be passed to `~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to automatically read the `date-obs` metadata. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> from sunpy.database.tables import entries_from_file >>> import sunpy.data.sample # doctest: +REMOTE_DATA >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA >>> len(entries) # doctest: +REMOTE_DATA 1 >>> entry = entries.pop() # doctest: +REMOTE_DATA >>> entry.instrument # doctest: +REMOTE_DATA 'SWAP' >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None) >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA 111 """ headers = fits.get_header(file) if isinstance(file, (str, six.text_type)): filename = file else: filename = getattr(file, 'name', None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in six.iteritems(header): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == '': value = str(value) elif key == 'KEYCOMMENTS': for k, v in six.iteritems(value): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) entry.hdu_index = headers.index(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == 'INSTRUME': entry.instrument = value elif key == 'WAVELNTH': if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to( nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ('DATE-END', 'DATE_END'): entry.observation_time_end = parse_time( value, _time_string_parse_format=time_string_parse_format) elif key in ('DATE-OBS', 'DATE_OBS'): entry.observation_time_start = parse_time( value, _time_string_parse_format=time_string_parse_format) yield entry
def _create(wlk, root, session): query = session.query(DatabaseEntry) for key, value in root.attrs.iteritems(): typ = key[0] if typ == "tag": criterion = TableTag.name.in_([value]) # `key[1]` is here the `inverted` attribute of the tag. That means # that if it is True, the given tag must not be included in the # resulting entries. if key[1]: query = query.filter(~DatabaseEntry.tags.any(criterion)) else: query = query.filter(DatabaseEntry.tags.any(criterion)) elif typ == "fitsheaderentry": key, val, inverted = value key_criterion = TableFitsHeaderEntry.key == key value_criterion = TableFitsHeaderEntry.value == val if inverted: query = query.filter( not_( and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any(value_criterion), ) ) ) else: query = query.filter( and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any(value_criterion), ) ) elif typ == "download time": start, end, inverted = value if inverted: query = query.filter(~DatabaseEntry.download_time.between(start, end)) else: query = query.filter(DatabaseEntry.download_time.between(start, end)) elif typ == "path": path, inverted = value if inverted: query = query.filter(or_(DatabaseEntry.path != path, DatabaseEntry.path == None)) else: query = query.filter(DatabaseEntry.path == path) elif typ == "wave": min_, max_, unit = value waveunit = Unit(unit) # convert min_ and max_ to nm from the unit `waveunit` wavemin = waveunit.to(nm, min_, equivalencies.spectral()) wavemax = waveunit.to(nm, max_, equivalencies.spectral()) query = query.filter(and_(DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax)) elif typ == "time": start, end, near = value query = query.filter( and_(DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start) ) else: if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(SUPPORTED_NONVSO_ATTRS): raise NotImplementedError("The attribute {0!r} is not yet supported to query a database.".format(typ)) query = query.filter_by(**{typ: value}) return query.all()
def entries_from_file(file, default_waveunit=None): """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) >>> len(entries) 1 >>> entry = entries.pop() >>> entry.instrument 'SWAP' >>> entry.observation_time_start, entry.observation_time_end (datetime.datetime(2012, 1, 1, 0, 16, 7, 836000), None) >>> entry.wavemin, entry.wavemax (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) 112 """ headers = fits.get_header(file) if isinstance(file, (str, unicode)): filename = file else: filename = getattr(file, "name", None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in header.iteritems(): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == "": value = str(value) elif key == "KEYCOMMENTS": for k, v in value.iteritems(): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == "INSTRUME": entry.instrument = value elif key == "WAVELNTH": if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to(nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ("DATE-END", "DATE_END"): entry.observation_time_end = parse_time(value) elif key in ("DATE-OBS", "DATE_OBS"): entry.observation_time_start = parse_time(value) yield entry
def entries_from_file(file, default_waveunit=None, time_string_parse_format=''): # Note: time_string_parse_format='' so that None won't be passed to Time.strptime # (which would make strptime freak out, if I remember correctly). """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. time_string_parse_format : str, optional Fallback timestamp format which will be passed to `~astropy.time.Time.strptime` if `sunpy.time.parse_time` is unable to automatically read the `date-obs` metadata. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> from sunpy.database.tables import entries_from_file >>> import sunpy.data.sample # doctest: +REMOTE_DATA >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA >>> len(entries) # doctest: +REMOTE_DATA 1 >>> entry = entries.pop() # doctest: +REMOTE_DATA >>> entry.instrument # doctest: +REMOTE_DATA 'SWAP' >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None) >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA 111 """ headers = fits.get_header(file) # This just checks for blank default headers # due to compression. for header in headers: if header == DEFAULT_HEADER: headers.remove(header) if isinstance(file, str): filename = file else: filename = getattr(file, 'name', None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in header.items(): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == '': value = str(value) elif key == 'KEYCOMMENTS': for k, v in value.items(): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) entry.hdu_index = headers.index(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == 'INSTRUME': entry.instrument = value elif key == 'WAVELNTH': if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to( nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ('DATE-END', 'DATE_END'): try: dt = parse_time(value).datetime except ValueError: dt = Time.strptime(value, time_string_parse_format).datetime entry.observation_time_end = dt elif key in ('DATE-OBS', 'DATE_OBS'): try: dt = parse_time(value).datetime except ValueError: dt = Time.strptime(value, time_string_parse_format).datetime entry.observation_time_start = dt yield entry