def test_no_time_error():
query = (a.Instrument('EVE'), a.Level(0))
with pytest.raises(ValueError) as excinfo:
Fido.search(*query)
assert all(str(a) in str(excinfo.value) for a in query)
query1 = (a.Instrument('EVE') & a.Level(0))
query2 = (a.Time("2012/1/1", "2012/1/2") & a.Instrument("AIA"))
with pytest.raises(ValueError) as excinfo:
Fido.search(query1 | query2)
assert all(str(a) in str(excinfo.value) for a in query1.attrs)
assert all(str(a) not in str(excinfo.value) for a in query2.attrs)
def test_unified_response():
start = parse_time("2012/1/1")
end = parse_time("2012/1/2")
qr = Fido.search(a.Instrument('EVE'), a.Level(0), a.Time(start, end))
assert qr.file_num == 2
strings = ['eve', 'SDO', start.strftime(TIMEFORMAT), end.strftime(TIMEFORMAT)]
assert all(s in qr._repr_html_() for s in strings)
def test_fido_indexing(queries):
query1, query2 = queries
# This is a work around for an aberration where the filter was not catching
# this.
assume(query1.attrs[1].start != query2.attrs[1].start)
res = Fido.search(query1 | query2)
assert len(res) == 2
assert len(res[0]) == 1
assert len(res[1]) == 1
aa = res[0, 0]
assert isinstance(aa, UnifiedResponse)
assert len(aa) == 1
assert len(aa.get_response(0)) == 1
aa = res[:, 0]
assert isinstance(aa, UnifiedResponse)
assert len(aa) == 2
assert len(aa.get_response(0)) == 1
aa = res[0, :]
assert isinstance(aa, UnifiedResponse)
assert len(aa) == 1
with pytest.raises(IndexError):
res[0, 0, 0]
with pytest.raises(IndexError):
res["saldkal"]
with pytest.raises(IndexError):
res[1.0132]
def test_unifiedresponse_slicing_reverse():
results = Fido.search(
a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
assert isinstance(results[::-1], UnifiedResponse)
assert len(results[::-1]) == len(results)
assert isinstance(results[0, ::-1], UnifiedResponse)
assert results[0, ::-1]._list[0] == results._list[0][::-1]
def test_save_path():
with tempfile.TemporaryDirectory() as target_dir:
qr = Fido.search(a.Instrument('EVE'), a.Time("2016/10/01", "2016/10/02"), a.Level(0))
files = Fido.fetch(qr, path=os.path.join(target_dir, "{instrument}"+os.path.sep+"{level}"))
for f in files:
assert target_dir in f
assert "eve{}0".format(os.path.sep) in f
def test_fido(mock_fetch):
qr = Fido.search(a.Time("2012/10/4", "2012/10/6"),
a.Instrument('noaa-indices'))
assert isinstance(qr, UnifiedResponse)
response = Fido.fetch(qr)
assert len(response) == qr._numfile
def test_fido(query):
qr = Fido.search(query)
client = qr.get_response(0).client
assert isinstance(qr, UnifiedResponse)
assert isinstance(client, eve.EVEClient)
response = Fido.fetch(qr)
assert len(response) == qr._numfile
def test_client_fetch_wrong_type(mock_fetch):
query = a.Time("2011/01/01", "2011/01/02") & a.Instrument("goes")
qr = Fido.search(query)
with pytest.raises(TypeError):
Fido.fetch(qr)
def test_no_wait_fetch():
qr = Fido.search(a.Instrument('EVE'),
a.Time("2016/10/01", "2016/10/02"),
a.Level(0))
res = Fido.fetch(qr, wait=False)
assert isinstance(res, DownloadResponse)
assert isinstance(res.wait(), list)
def test_levels(time):
"""
Test the correct handling of level 0 / 1.
The default should be level 1 from VSO, level 0 comes from EVEClient.
"""
eve_a = a.Instrument('EVE')
qr = Fido.search(time, eve_a)
client = qr.get_response(0).client
assert isinstance(client, VSOClient)
qr = Fido.search(time, eve_a, a.Level(0))
client = qr.get_response(0).client
assert isinstance(client, eve.EVEClient)
qr = Fido.search(time, eve_a, a.Level(0) | a.Level(1))
clients = {type(a.client) for a in qr.responses}
assert clients.symmetric_difference({VSOClient, eve.EVEClient}) == set()
def test_responses():
results = Fido.search(
a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
for i, resp in enumerate(results.responses):
assert isinstance(resp, QueryResponse)
assert i + 1 == len(results)
def test_save_path(tmpdir):
qr = Fido.search(a.Instrument('EVE'), a.Time("2016/10/01", "2016/10/02"), a.Level(0))
# Test when path is str
files = Fido.fetch(qr, path=str(tmpdir / "{instrument}" / "{level}"))
for f in files:
assert str(tmpdir) in f
assert "eve{}0".format(os.path.sep) in f
def test_repr():
results = Fido.search(
a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
rep = repr(results)
rep = rep.split('\n')
# 6 header lines, the results table and two blank lines at the end
assert len(rep) == 7 + len(list(results.responses)[0]) + 2
def test_add_entries_from_fido_search_result_JSOC_client(database):
assert len(database) == 0
search_result = Fido.search(
net_attrs.jsoc.Time('2014-01-01T00:00:00', '2014-01-01T01:00:00'),
net_attrs.jsoc.Series('hmi.m_45s'),
net_attrs.jsoc.Notify("*****@*****.**")
)
with pytest.raises(ValueError):
database.add_from_fido_search_result(search_result)
def test_multiple_match():
"""
Using the builtin clients a multiple match is not possible so we create a
dummy class.
"""
new_registry = copy.deepcopy(Fido.registry)
Fido.registry = new_registry
class DummyClient():
@classmethod
def _can_handle_query(cls, *query):
return True
Fido.registry.update({DummyClient: DummyClient._can_handle_query})
with pytest.raises(MultipleMatchError):
Fido.search(a.Time("2016/10/1", "2016/10/2"), a.Instrument('lyra'))
Fido.registry = CLIENTS
def test_save_path_pathlib(tmpdir):
qr = Fido.search(a.Instrument('EVE'), a.Time("2016/10/01", "2016/10/02"), a.Level(0))
# Test when path is pathlib.Path
target_dir = tmpdir.mkdir("down")
path = pathlib.Path(target_dir, "{instrument}", "{level}")
files = Fido.fetch(qr, path=path)
for f in files:
assert target_dir.strpath in f
assert "eve{}0".format(os.path.sep) in f
def test_save_path_pathlib():
pathlib = pytest.importorskip('pathlib')
qr = Fido.search(a.Instrument('EVE'), a.Time("2016/10/01", "2016/10/02"), a.Level(0))
# Test when path is pathlib.Path
with tempfile.TemporaryDirectory() as target_dir:
path = pathlib.Path(target_dir, "{instrument}", "{level}")
files = Fido.fetch(qr, path=path)
for f in files:
assert target_dir in f
assert "eve{}0".format(os.path.sep) in f
def test_fido_iter(queries):
query1, query2 = queries
# This is a work around for an aberration where the filter was not catching
# this.
assume(query1.attrs[1].start != query2.attrs[1].start)
res = Fido.search(query1 | query2)
for resp in res:
assert isinstance(resp, QueryResponse)
def test_entries_from_fido_search_result_JSOC():
search_result = Fido.search(
net_attrs.jsoc.Time('2014-01-01T00:00:00', '2014-01-01T01:00:00'),
net_attrs.jsoc.Series('hmi.m_45s'),
net_attrs.jsoc.Notify("*****@*****.**")
)
with pytest.raises(ValueError):
# Using list() here is important because the
# entries_from_fido_search_result function uses yield.
# list() uses the generator to run the function body.
list(entries_from_fido_search_result(search_result))
def test_fido(mock_wait, mock_search, mock_enqueue):
qr1 = Fido.search(Time('2012/10/4', '2012/10/6'),
Instrument('noaa-indices'))
Fido.fetch(qr1, path="/some/path/{file}")
# Here we assert that the `fetch` function has called the parfive
# Downloader.enqueue_file method with the correct arguments. Everything
# that happens after this point should either be tested in the
# GenericClient tests or in parfive itself.
assert mock_enqueue.called_once_with(("ftp://ftp.swpc.noaa.gov/pub/weekly/RecentIndices.txt",
"/some/path/RecentIndices.txt"))
def test_repr(query):
res = Fido.search(query)
for rep_meth in (res.__repr__, res.__str__, res._repr_html_):
if len(res) == 1:
assert "Provider" in rep_meth()
assert "Providers" not in rep_meth()
else:
assert "Provider" not in rep_meth()
assert "Providers" in rep_meth()
def fido_search_result():
# A search query with responses from all instruments
# No JSOC query
return Fido.search(
net_attrs.Time("2012/1/1", "2012/1/2"),
net_attrs.Instrument('lyra') | net_attrs.Instrument('eve') |
net_attrs.Instrument('XRS') | net_attrs.Instrument('noaa-indices') |
net_attrs.Instrument('noaa-predict') |
(net_attrs.Instrument('norh') & net_attrs.Wavelength(17*units.GHz)) |
net_attrs.Instrument('rhessi') |
(net_attrs.Instrument('EVE') & net_attrs.Level(0))
)
def test_vso_errors_with_second_client(mock_download_all):
query = a.Time("2011/01/01", "2011/01/02") & (a.Instrument("goes") | a.Instrument("EIT"))
qr = Fido.search(query)
res = Fido.fetch(qr)
assert len(res.errors) == 1
assert len(res) != qr.file_num
# Assert that all the XRSClient records are in the output.
for resp in qr.responses:
if isinstance(resp, XRSClient):
assert len(resp) == len(res)
##############################################################################
# Let's select a date (yyyy-mm-dd) for which we will be downloading files.
day = parse_time("2017-01-25")
##############################################################################
# We will select the entire day as our timerange.
start_time = day
end_time = day + TimeDelta(23*u.hour + 59*u.minute + 59*u.second)
##############################################################################
# Send the search query.
results = Fido.search(a.Time(start_time, end_time),
a.Instrument('HMI') & a.vso.Physobs("LOS_magnetic_field"),
a.vso.Sample(60 * u.second))
##############################################################################
# We will only download the first file for the day. For that we use fido
# indexing on the search results which will return the first file for the day.
result = results[0, 0]
##############################################################################
# Download the file. The `fetch` method returns a list of filenames. As we
# used indexing to get the first file of the day, the list contains one
# filename.
file_name = Fido.fetch(result)
def test_fido(time, instrument):
qr = Fido.search(time, instrument)
assert isinstance(qr, UnifiedResponse)
response = Fido.fetch(qr)
assert len(response) == qr._numfile
def test_fetch():
qr = Fido.search(a.Instrument('EVE'), a.Time("2016/10/01", "2016/10/02"),
a.Level(0))
res = Fido.fetch(qr)
assert isinstance(res, Results)
def test_fido_mock(mock_get_observing_summary_dbase_file,
mock_parse_observing_summary_dbase_file,
mock_get_base_url):
qr = Fido.search(a.Time('2003-11-01', '2003-11-03'), a.Instrument('rhessi'))
assert isinstance(qr, UnifiedResponse)
assert qr._numfile == 3
def test_path():
results = Fido.search(a.Time("2012/1/1", "2012/1/5"), a.Instrument.lyra)
Fido.fetch(results, path="notapath/{file}")
# The first step is to download some data, we are going to get an image from
# early 2011 when the STEREO spacecraft were roughly 90 deg seperated from the
# Earth.
stereo = (a.vso.Source('STEREO_B') &
a.Instrument('EUVI') &
a.Time('2011-01-01', '2011-01-01T00:10:00'))
aia = (a.Instrument('AIA') &
a.vso.Sample(24 * u.hour) &
a.Time('2011-01-01', '2011-01-02'))
wave = a.Wavelength(30 * u.nm, 31 * u.nm)
res = Fido.search(wave, aia | stereo)
###############################################################################
# The results from VSO query:
print(res)
###############################################################################
# Download the files:
files = Fido.fetch(res)
print(files)
###############################################################################
# Create a dictionary with the two maps, cropped down to full disk.
maps = {m.detector: m.submap(SkyCoord([-1100, 1100]*u.arcsec,
def test_unifiedresponse_slicing():
results = Fido.search(a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
assert isinstance(results[0:2], UnifiedResponse)
assert isinstance(results[0], UnifiedResponse)
def test_online_fido(query):
unifiedresp = Fido.search(query)
check_response(query, unifiedresp)
def test_unifiedresponse_slicing_reverse():
results = Fido.search(a.Time("2012/1/1", "2012/1/5"), a.Instrument.lyra)
assert isinstance(results[::-1], UnifiedResponse)
assert len(results[::-1]) == len(results[::1])
assert isinstance(results[0, ::-1], QueryResponseTable)
assert all(results[0][::-1] == results[0, ::-1])
ts_goes = sunpy.timeseries.TimeSeries(sunpy.data.sample.GOES_XRS_TIMESERIES, source='XRS')
ts_lyra = sunpy.timeseries.TimeSeries(sunpy.data.sample.LYRA_LEVEL3_TIMESERIES, source='LYRA')
ts_noaa_ind = sunpy.timeseries.TimeSeries(
sunpy.data.sample.NOAAINDICES_TIMESERIES, source='NOAAIndices')
ts_noaa_pre = sunpy.timeseries.TimeSeries(
sunpy.data.sample.NOAAPREDICT_TIMESERIES, source='NOAAPredictIndices')
ts_norh = sunpy.timeseries.TimeSeries(sunpy.data.sample.NORH_TIMESERIES, source='NoRH')
ts_rhessi = sunpy.timeseries.TimeSeries(sunpy.data.sample.RHESSI_TIMESERIES, source='RHESSI')
ts_gbm = sunpy.timeseries.TimeSeries(sunpy.data.sample.GBM_TIMESERIES, source='GBMSummary')
# Note: for some FITS files a source can be determined implicitly, however it
# is good practice to delcare it explicitly when possible.
##############################################################################
# You can create a list of TimeSeries objects by using multiple files. First
# however, we shall download these files using `Fido`.
goes = Fido.search(a.Time("2012/06/01", "2012/06/04"), a.Instrument("XRS"))
goes_files = Fido.fetch(goes)
# Using these new files you get a list:
lis_goes_ts = sunpy.timeseries.TimeSeries(goes_files[:2], source='XRS')
lis_goes_ts = sunpy.timeseries.TimeSeries(goes_files, source='XRS')
# Using concatenate=True kwarg you can merge the files into one TimeSeries:
combined_goes_ts = sunpy.timeseries.TimeSeries(goes_files, source='XRS', concatenate=True)
combined_goes_ts.peek()
# Note: ATM we only accept TimeSeries of a single class being created together
# with the factory. The issue is that several source filetypes don't contain
# metadata that enables us to reliably implicitly gather the source and ATM the
# source is given as a single keyword argument for simplicity. But you can merge
# different TimeSeries classes using concatenate.
# Debate: are we OK for one source at a time?
def test_fido_indexing(queries):
query1, query2 = queries
# This is a work around for an aberration where the filter was not catching
# this.
assume(query1.attrs[1].start != query2.attrs[1].start)
res = Fido.search(query1 | query2)
assert len(res) == 2
assert isinstance(res[1:], UnifiedResponse)
assert len(res[1:]) == 1
assert isinstance(res[0:1], UnifiedResponse)
assert len(res[0:1]) == 1
assert isinstance(res[1:, 0], UnifiedResponse)
assert len(res[1:, 0]) == 1
assert isinstance(res[0:1, 0], UnifiedResponse)
assert len(res[0:1, 0]) == 1
assert isinstance(res[0][0], QueryResponseRow)
assert isinstance(res[1][0], QueryResponseRow)
assert isinstance(res[1, 0:1], QueryResponseTable)
aa = res[0, 0]
assert isinstance(aa, QueryResponseRow)
aa = res[0, 'Instrument']
assert isinstance(aa, QueryResponseColumn)
aa = res[:, 'Instrument']
assert isinstance(aa, UnifiedResponse)
for table in aa:
assert len(table.columns) == 1
aa = res[0, ('Instrument', )]
assert isinstance(aa, QueryResponseTable)
for table in aa:
assert len(table.columns) == 1
aa = res[:, 0]
assert isinstance(aa, UnifiedResponse)
assert len(aa) == 2
assert len(aa[0]) == 1
aa = res[0, :]
assert isinstance(aa, QueryResponseTable)
aa = res[0, 1:]
assert isinstance(aa, QueryResponseTable)
if len(res.keys()) == len(res):
aa = res[res.keys()[0], 1:]
assert isinstance(aa, QueryResponseTable)
aa = res[res.keys()[0], 'Instrument']
assert isinstance(aa, QueryResponseColumn)
with pytest.raises(IndexError):
res[0, 0, 0]
with pytest.raises(IndexError):
res["saldkal"]
with pytest.raises(IndexError):
res[1.0132]
if isinstance(res, UnifiedResponse):
assert len(res) != 1
def test_client_fetch_wrong_type(mock_fetch):
query = a.Time("2011/01/01", "2011/01/02") & a.Instrument.goes
qr = Fido.search(query)
with pytest.raises(TypeError):
Fido.fetch(qr)
A simple example showing how to download data from the VSO with Fido.
"""
###############################################################################
# Fido is the primary interface to search for and download data and
# will search the VSO if appropriate. First import it and the search
# attributes.
import astropy.units as u
from sunpy.net import Fido, attrs as a
###############################################################################
# We could ask for all SOHO/EIT data between January 1st and 2nd, 2001.
attrs_time = a.Time('2005/01/01 00:10', '2005/01/01 00:15')
result = Fido.search(attrs_time, a.Instrument('eit'))
###############################################################################
# Let's inspect the result
print(result)
###############################################################################
# Now lets download this query. If we don't provide a path it will download the
# file into the sunpy data directory.
downloaded_files = Fido.fetch(result)
###############################################################################
# You can check where the file was downloaded to.
def test_no_match():
with pytest.raises(DrmsQueryError):
Fido.search(a.Time("2016/10/01", "2016/10/02"), a.jsoc.Series("bob"),
a.vso.Sample(10 * u.s))
import astropy.units as u
import matplotlib.pyplot as plt
import sunpy.map
from sunpy.net import Fido, attrs as a
###############################################################################
# Now we will download some data with `sunpy.net.Fido`.
# A `Fido.search` requires us to specify a `~sunpy.net.attr.Time`,
# `~sunpy.net.attr.Sample`, `~sunpy.net.attr.Instrument`
# and the `~sunpy.net.attr.vso.Physobs`.
# We set a time range from ``2015/11/04 12:00:00`` to ``2015/11/04 12:10:00``
# for HMI ``LOS_magnetic_field`` with the images spaced every 720 seconds.
result = Fido.search(a.Time('2018/11/04 12:00:00', '2018/11/04 12:10:00'),
a.Instrument('hmi'),
a.Sample(720*u.s),
a.vso.Physobs('LOS_magnetic_field'))
###############################################################################
# Now we can see what results we obtained from our search.
# Notice we have two files. One is the full disk image we plan to display
# and the other is a synoptic version of said image.
print(result)
###############################################################################
# Once we are happy with the results obtained from the search.
# We can download the data with `Fido.fetch` .
# In this case we only want one file so we can index the result.
# A `Fido` result can be from several clients, so we have to index the first
# client and then index the first result.
def test_unclosedSocket_warning():
with pytest.warns(None):
attrs_time = a.Time('2005/01/01 00:10', '2005/01/01 00:15')
result = Fido.search(attrs_time, a.Instrument.eit)
Fido.fetch(result)
def test_fido_mock(mock_get_observing_summary_dbase_file,
mock_parse_observing_summary_dbase_file, mock_get_base_url):
qr = Fido.search(a.Time('2003-11-01', '2003-11-03'), a.Instrument.rhessi)
assert isinstance(qr, UnifiedResponse)
assert qr._numfile == 3
def test_unifiedresponse_slicing_reverse():
results = Fido.search(a.Time("2012/1/1", "2012/1/5"), a.Instrument("lyra"))
assert isinstance(results[::-1], UnifiedResponse)
assert len(results[::-1]) == len(results)
assert isinstance(results[0, ::-1], UnifiedResponse)
assert results[0, ::-1]._list[0] == results._list[0][::-1]
