def query(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> client.query(
... vso.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... vso.Instrument('eit') | vso.Instrument('aia')
... )
Returns
-------
out : :py:class:`QueryResult` (enhanced list) of matched items. Return value of same type as the one of :py:meth:`VSOClient.query`.
"""
query = and_(*query)
responses = []
for block in walker.create(query, self.api):
try:
responses.append(
self.api.service.Query(
self.make('QueryRequest', block=block)))
except TypeNotFound:
pass
except Exception as ex:
response = QueryResponse.create(self.merge(responses))
response.add_error(ex)
return QueryResponse.create(self.merge(responses))
def query(self, *query):
""" Furtherly reduce the query response by matching it against
another query, e.g. response.query(attrs.Instrument('aia')). """
query = and_(*query)
return QueryResponse(
attrs.filter_results(query, self), self.queryresult
)
def test_and_nesting():
a = attr.and_(
attrs.Level(0),
attr.AttrAnd(
(attrs.Instrument('EVE'), attrs.Time("2012/1/1", "2012/01/02"))))
# Test that the nesting has been removed.
assert len(a.attrs) == 3
def query(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> client.query(
... vso.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... vso.Instrument('eit') | vso.Instrument('aia')
... )
Returns
-------
out : :py:class:`QueryResult` (enhanced list) of matched items. Return value of same type as the one of :py:meth:`VSOClient.query`.
"""
query = and_(*query)
responses = []
for block in walker.create(query, self.api):
try:
responses.append(self.api.service.Query(self.make("QueryRequest", block=block)))
except TypeNotFound:
pass
except Exception as ex:
response = QueryResponse.create(self.merge(responses))
response.add_error(ex)
return QueryResponse.create(self.merge(responses))
def search(self, *query):
""" Furtherly reduce the query response by matching it against
another query, e.g. response.search(attrs.Instrument('aia')). """
query = and_(*query)
return QueryResponse(
attrs.filter_results(query, self), self.queryresult
)
def search(self, *query):
"""
Query for data in form of multiple parameters.
Examples
--------
Query for LYRA timeseries data for the time range ('2012/3/4','2012/3/6')
>>> from sunpy.net import Fido, attrs as a
>>> import astropy.units as u
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'), a.Instrument.lyra) # doctest: +REMOTE_DATA
Query for data from Nobeyama Radioheliograph and RHESSI
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
... (a.Instrument.norh & a.Wavelength(17*u.GHz)) | a.Instrument.rhessi) # doctest: +REMOTE_DATA
Query for 304 Angstrom SDO AIA data with a cadence of 10 minutes
>>> import astropy.units as u
>>> from sunpy.net import Fido, attrs as a
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
... a.Instrument.aia,
... a.Wavelength(304*u.angstrom, 304*u.angstrom),
... a.Sample(10*u.minute)) # doctest: +REMOTE_DATA
Parameters
----------
query : `sunpy.net.vso.attrs`, `sunpy.net.jsoc.attrs`
A query consisting of multiple parameters which define the
requested data. The query is specified using attributes from the
VSO and the JSOC. The query can mix attributes from the VSO and
the JSOC.
Returns
-------
`sunpy.net.fido_factory.UnifiedResponse`
Container of responses returned by clients servicing query.
Notes
-----
The conjunction 'and' transforms query into disjunctive normal form
ie. query is now of form A & B or ((A & B) | (C & D))
This helps in modularising query into parts and handling each of the
parts individually.
"""
query = attr.and_(*query)
results = query_walker.create(query, self)
# If we have searched the VSO but no results were returned, but another
# client generated results, we drop the empty VSO results for tidiness.
# This is because the VSO _can_handle_query is very broad because we
# don't know the full list of supported values we can search for (yet).
if len(results) > 1:
vso_results = list(filter(lambda r: isinstance(r, vso.VSOQueryResponseTable), results))
for vres in vso_results:
if len(vres) == 0:
results.remove(vres)
return UnifiedResponse(*results)
def search_metadata(self, *query, **kwargs):
"""
Get the metadata of all the files obtained in a search query.
Builds a jsoc query, similar to query method, and takes similar inputs.
Complex queries to be easily formed using logical operators such as
``&`` and ``|``, in the same way as the query function.
Parameters
----------
query : a variable number of `~sunpy.net.jsoc.attrs`
as parameters, which are chained together using
the ``AND`` (``&``) operator.
Returns
-------
res : `~pandas.DataFrame` object
A collection of metadata of all the files.
"""
query = and_(*query)
blocks = []
res = pd.DataFrame()
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
iargs.update({'meta': True})
blocks.append(iargs)
res = res.append(self._lookup_records(iargs))
return res
def download(self, *query, **kwargs):
"""download(*query, client=sunpy.net.vso.VSOClient(), path=None, progress=False)
Search for data using the VSO interface (see
:meth:`sunpy.net.vso.VSOClient.query`). If querying the VSO results in
no data, no operation is performed. Concrete, this means that no entry
is added to the database and no file is downloaded. Otherwise, the
retrieved search result is used to download all files that belong to
this search result. After that, all the gathered information (the one
from the VSO query result and the one from the downloaded FITS files)
is added to the database in a way that each FITS header is represented
by one database entry.
"""
if not query:
raise TypeError('at least one attribute required')
client = kwargs.pop('client', None)
path = kwargs.pop('path', None)
progress = kwargs.pop('progress', False)
if kwargs:
k, v = kwargs.popitem()
raise TypeError('unexpected keyword argument {0!r}'.format(k))
if client is None:
client = VSOClient()
qr = client.query(*query)
# don't do anything if querying the VSO results in no data
if not qr:
return
entries = list(
self._download_and_collect_entries(qr, client, path, progress))
dump = serialize.dump_query(and_(*query))
(dump_exists, ), = self.session.query(
exists().where(tables.JSONDump.dump == tables.JSONDump(dump).dump))
if dump_exists:
# dump already exists in table jsondumps -> edit instead of add
# update all entries with the fileid `entry.fileid`
for entry in entries:
old_entry = self.session.query(tables.DatabaseEntry).filter_by(
fileid=entry.fileid).first()
if old_entry is not None:
attrs = [
'source', 'provider', 'physobs',
'observation_time_start', 'observation_time_end',
'instrument', 'size', 'wavemin', 'wavemax',
'download_time'
]
kwargs = dict((k, getattr(entry, k)) for k in attrs)
cmd = commands.EditEntry(old_entry, **kwargs)
if self._enable_history:
self._command_manager.do(cmd)
else:
cmd()
else:
self.add_many(entries)
# serialize the query and save the serialization in the database
# for two reasons:
# 1. to avoid unnecessary downloading in future calls of
# ``fetch``
# 2. to know whether to add or to edit entries in future calls of
# ``download`` (this method)
self.session.add(tables.JSONDump(dump))
def download(self, *query, **kwargs):
"""download(*query, client=sunpy.net.vso.VSOClient(), path=None, progress=False)
Search for data using the VSO interface (see
:meth:`sunpy.net.vso.VSOClient.query`). If querying the VSO results in
no data, no operation is performed. Concrete, this means that no entry
is added to the database and no file is downloaded. Otherwise, the
retrieved search result is used to download all files that belong to
this search result. After that, all the gathered information (the one
from the VSO query result and the one from the downloaded FITS files)
is added to the database in a way that each FITS header is represented
by one database entry.
"""
if not query:
raise TypeError('at least one attribute required')
client = kwargs.pop('client', None)
path = kwargs.pop('path', None)
progress = kwargs.pop('progress', False)
if kwargs:
k, v = kwargs.popitem()
raise TypeError('unexpected keyword argument {0!r}'.format(k))
if client is None:
client = VSOClient()
qr = client.query(*query)
# don't do anything if querying the VSO results in no data
if not qr:
return
entries = list(self._download_and_collect_entries(
qr, client, path, progress))
dump = serialize.dump_query(and_(*query))
(dump_exists,), = self.session.query(
exists().where(tables.JSONDump.dump == tables.JSONDump(dump).dump))
if dump_exists:
# dump already exists in table jsondumps -> edit instead of add
# update all entries with the fileid `entry.fileid`
for entry in entries:
old_entry = self.session.query(
tables.DatabaseEntry).filter_by(fileid=entry.fileid).first()
if old_entry is not None:
attrs = [
'source', 'provider', 'physobs',
'observation_time_start', 'observation_time_end',
'instrument', 'size', 'wavemin', 'wavemax',
'download_time']
kwargs = dict((k, getattr(entry, k)) for k in attrs)
cmd = commands.EditEntry(old_entry, **kwargs)
if self._enable_history:
self._command_manager.do(cmd)
else:
cmd()
else:
self.add_many(entries)
# serialize the query and save the serialization in the database
# for two reasons:
# 1. to avoid unnecessary downloading in future calls of
# ``fetch``
# 2. to know whether to add or to edit entries in future calls of
# ``download`` (this method)
self.session.add(tables.JSONDump(dump))
def test_and_nesting():
a1 = SA1(1)
a2 = SA2(2)
a3 = SA3(3)
a = attr.and_(a1, attr.AttrAnd((a2, a3)))
# Test that the nesting has been removed.
assert len(a.attrs) == 3
def test_and_nesting():
a1 = SA1(1)
a2 = SA2(2)
a3 = SA3(3)
a = attr.and_(a1, attr.AttrAnd((a2, a3)))
# Test that the nesting has been removed.
assert len(a.attrs) == 3
def query(self, *query, **kwargs):
"""
query(*query[, sortby])
Send the given query to the database and return a list of
database entries that satisfy all of the given attributes.
Apart from the attributes supported by the VSO interface, the following
attributes are supported:
- :class:`sunpy.database.attrs.Starred`
- :class:`sunpy.database.attrs.Tag`
- :class:`sunpy.database.attrs.Path`
- :class:`sunpy.database.attrs.DownloadTime`
- :class:`sunpy.database.attrs.FitsHeaderEntry`
An important difference to the VSO attributes is that these attributes
may also be used in negated form using the tilde ~ operator.
Parameters
----------
query : list
A variable number of attributes that are chained together via the
boolean AND operator. The | operator may be used between attributes
to express the boolean OR operator.
sortby : str, optional
The column by which to sort the returned entries. The default is to
sort by the start of the observation. See the attributes of
:class:`sunpy.database.tables.DatabaseEntry` for a list of all
possible values.
Raises
------
TypeError
if no attribute is given or if some keyword argument other than
'sortby' is given.
Examples
--------
The query in the following example searches for all non-starred entries
with the tag 'foo' or 'bar' (or both).
>>> database.query(~attrs.Starred(), attrs.Tag('foo') | attrs.Tag('bar'))
"""
if not query:
raise TypeError('at least one attribute required')
sortby = kwargs.pop('sortby', 'observation_time_start')
if kwargs:
k, v = kwargs.popitem()
raise TypeError('unexpected keyword argument {0!r}'.format(k))
return sorted(
walker.create(and_(*query), self.session),
key=operator.attrgetter(sortby))
def search_metadata(self, *query, **kwargs):
"""
Get the metadata of all the files obtained in a search query.
Builds a jsoc query, similar to query method, and takes similar inputs.
Complex queries to be easily formed using logical operators such as
``&`` and ``|``, in the same way as the query function.
Parameters
----------
query : a variable number of `~sunpy.net.jsoc.attrs`
as parameters, which are chained together using
the ``AND`` (``&``) operator.
Returns
-------
res : `~pandas.DataFrame` object
A collection of metadata of all the files.
Example
-------
Request metadata or all all AIA 304 image data between 2014-01-01T00:00 and
2014-01-01T01:00.
Since, the function only performs a lookdata, and does not make a proper export
request, attributes like Segment need not be passed::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> metadata = client.search_metadata(
... a.Time('2014-01-01T00:00:00', '2014-01-01T00:01:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(304*u.AA)) # doctest: +REMOTE_DATA
>>> print(metadata[['T_OBS', 'WAVELNTH']]) # doctest: +REMOTE_DATA
T_OBS WAVELNTH
aia.lev1_euv_12s[2014-01-01T00:00:01Z][304] 2014-01-01T00:00:08.57Z 304
aia.lev1_euv_12s[2014-01-01T00:00:13Z][304] 2014-01-01T00:00:20.58Z 304
aia.lev1_euv_12s[2014-01-01T00:00:25Z][304] 2014-01-01T00:00:32.57Z 304
aia.lev1_euv_12s[2014-01-01T00:00:37Z][304] 2014-01-01T00:00:44.58Z 304
aia.lev1_euv_12s[2014-01-01T00:00:49Z][304] 2014-01-01T00:00:56.57Z 304
aia.lev1_euv_12s[2014-01-01T00:01:01Z][304] 2014-01-01T00:01:08.59Z 304
"""
query = and_(*query)
blocks = []
res = pd.DataFrame()
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
iargs.update({'meta': True})
blocks.append(iargs)
res = res.append(self._lookup_records(iargs))
return res
def search_metadata(self, *query, **kwargs):
"""
Get the metadata of all the files obtained in a search query.
Builds a jsoc query, similar to query method, and takes similar inputs.
Complex queries to be easily formed using logical operators such as
``&`` and ``|``, in the same way as the query function.
Parameters
----------
query : a variable number of `~sunpy.net.jsoc.attrs`
as parameters, which are chained together using
the ``AND`` (``&``) operator.
Returns
-------
res : `~pandas.DataFrame` object
A collection of metadata of all the files.
Example
-------
Request metadata or all all AIA 304 image data between 2014-01-01T00:00 and
2014-01-01T01:00.
Since, the function only performs a lookdata, and does not make a proper export
request, attributes like Segment need not be passed::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> metadata = client.search_metadata(
... a.Time('2014-01-01T00:00:00', '2014-01-01T00:01:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(304*u.AA)) # doctest: +REMOTE_DATA
>>> print(metadata[['T_OBS', 'WAVELNTH']]) # doctest: +REMOTE_DATA
T_OBS WAVELNTH
aia.lev1_euv_12s[2014-01-01T00:00:01Z][304] 2014-01-01T00:00:08.57Z 304
aia.lev1_euv_12s[2014-01-01T00:00:13Z][304] 2014-01-01T00:00:20.58Z 304
aia.lev1_euv_12s[2014-01-01T00:00:25Z][304] 2014-01-01T00:00:32.57Z 304
aia.lev1_euv_12s[2014-01-01T00:00:37Z][304] 2014-01-01T00:00:44.58Z 304
aia.lev1_euv_12s[2014-01-01T00:00:49Z][304] 2014-01-01T00:00:56.57Z 304
aia.lev1_euv_12s[2014-01-01T00:01:01Z][304] 2014-01-01T00:01:08.59Z 304
"""
query = and_(*query)
blocks = []
res = pd.DataFrame()
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
iargs.update({'meta': True})
blocks.append(iargs)
res = res.append(self._lookup_records(iargs))
return res
def query_and(draw,
stattrs=st.lists(st.sampled_from(_supported_attr_types()),
min_size=1,
unique=True)):
"""
Generate a AttrAnd query.
"""
attr_types = draw(stattrs)
query_attrs = list(map(draw, map(st.from_type, attr_types)))
assume(not (len(query_attrs) == 1 and isinstance(query_attrs[0], a.Time)))
return attr.and_(*query_attrs)
def offline_query(draw, instrument=offline_instruments()):
"""
Strategy for any valid offline query
"""
query = draw(instrument)
# If we have AttrAnd then we don't have GOES
if isinstance(query, a.Instrument) and query.value == 'goes':
query &= draw(goes_time())
else:
query = attr.and_(query, draw(time_attr()))
return query
def offline_query(draw, instrument=offline_instruments()):
"""
Strategy for any valid offline query
"""
query = draw(instrument)
# If we have AttrAnd then we don't have GOES
if isinstance(query, a.Instrument) and query.value == 'goes':
query &= draw(goes_time())
else:
query = attr.and_(query, draw(time_attr()))
return query
def search(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso, attrs as a
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> client.search(
... a.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... a.Instrument('eit') | a.Instrument('aia')) # doctest: +REMOTE_DATA
<QTable length=5>
Start Time [1] End Time [1] Source ... Type Wavelength [2]
... Angstrom
str19 str19 str4 ... str8 float64
------------------- ------------------- ------ ... -------- --------------
2010-01-01 00:00:08 2010-01-01 00:00:20 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:12:08 2010-01-01 00:12:20 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:24:10 2010-01-01 00:24:22 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:36:08 2010-01-01 00:36:20 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:48:09 2010-01-01 00:48:21 SOHO ... FULLDISK 195.0 .. 195.0
Returns
-------
out : :py:class:`QueryResult` (enhanced list)
Matched items. Return value is of same type as the one of
:py:meth:`VSOClient.search`.
"""
query = and_(*query)
QueryRequest = self.api.get_type('VSO:QueryRequest')
VSOQueryResponse = self.api.get_type('VSO:QueryResponse')
responses = []
for block in walker.create(query, self.api):
try:
query_response = self.api.service.Query(
QueryRequest(block=block)
)
for resp in query_response:
if resp["error"]:
warnings.warn(resp["error"], SunpyUserWarning)
responses.append(
VSOQueryResponse(query_response)
)
except Exception as ex:
response = QueryResponse.create(self.merge(responses))
response.add_error(ex)
return QueryResponse.create(self.merge(responses))
def search(self, *query, **kwargs):
query = and_(*query)
queries = walker.create(query)
results = []
for query_parameters in queries:
results.append(self._do_search(query_parameters))
table = astropy.table.vstack(results)
qrt = QueryResponseTable(table, client=self)
qrt['Filesize'] = (qrt['Filesize'] * u.byte).to(u.Mbyte).round(3)
qrt.hide_keys = ['Data item ID', 'Filename']
return qrt
def query(self, *query):
query = attr.and_(*query)
data = attrs.walker.create(query, {})
ndata = []
for elem in data:
new = self.default.copy()
new.update(elem)
ndata.append(new)
if len(ndata) == 1:
return self._download(ndata[0])
else:
return self.merge(self._download(data) for data in ndata)
def search(self, *query, **kwargs):
"""
Search for datasets provided by the Space Physics Data Facility.
"""
query = and_(*query)
queries = walker.create(query)
results = []
for query_parameters in queries:
results.append(self._do_search(query_parameters))
table = astropy.table.vstack(results)
qrt = QueryResponseTable(table, client=self)
qrt.hide_keys = ['URL']
return qrt
def search(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> client.search(
... vso.attrs.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... vso.attrs.Instrument('eit') | vso.attrs.Instrument('aia')) # doctest: +REMOTE_DATA
<QTable length=5>
Start Time [1] End Time [1] Source ... Type Wavelength [2]
... Angstrom
str19 str19 str4 ... str8 float64
------------------- ------------------- ------ ... -------- --------------
2010-01-01 00:00:08 2010-01-01 00:00:20 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:12:08 2010-01-01 00:12:20 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:24:10 2010-01-01 00:24:22 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:36:08 2010-01-01 00:36:20 SOHO ... FULLDISK 195.0 .. 195.0
2010-01-01 00:48:09 2010-01-01 00:48:21 SOHO ... FULLDISK 195.0 .. 195.0
Returns
-------
out : :py:class:`QueryResult` (enhanced list)
Matched items. Return value is of same type as the one of
:py:meth:`VSOClient.search`.
"""
query = and_(*query)
QueryRequest = self.api.get_type('VSO:QueryRequest')
VSOQueryResponse = self.api.get_type('VSO:QueryResponse')
responses = []
for block in walker.create(query, self.api):
try:
responses.append(
VSOQueryResponse(self.api.service.Query(
QueryRequest(block=block)
))
)
except Exception as ex:
response = QueryResponse.create(self.merge(responses))
response.add_error(ex)
return QueryResponse.create(self.merge(responses))
def split_database(source_database, destination_database, *query_string):
"""
Queries the source database with the query string, and moves the
matched entries to the destination database. When this function is
called, the `~sunpy.database.Database.undo` feature is disabled for both databases.
Parameters
----------
source_database : `~sunpy.database.database.Database`
A SunPy `~Database` object. This is the database on which the queries
will be made.
destination_database : `~sunpy.database.database.Database`
A SunPy `~Database` object. This is the database to which the matched
entries will be moved.
query_string : `list`
A variable number of attributes that are chained together via the
boolean AND operator. The | operator may be used between attributes
to express the boolean OR operator.
Examples
--------
The function call in the following example moves those entries from
database1 to database2 which have `~sunpy.net.vso.attrs.Instrument` = 'AIA' or
'ERNE'.
>>> from sunpy.database import Database, split_database
>>> from sunpy.database.tables import display_entries
>>> from sunpy.net import vso
>>> database1 = Database('sqlite:///:memory:')
>>> database2 = Database('sqlite:///:memory:')
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.search(vso.attrs.Time('2011-05-08', '2011-05-08 00:00:05')) # doctest: +REMOTE_DATA
>>> database1.add_from_vso_query_result(qr) # doctest: +REMOTE_DATA
>>> database1, database2 = split_database(database1, database2,
... vso.attrs.Instrument('AIA') | vso.attrs.Instrument('ERNE')) # doctest: +REMOTE_DATA
"""
query_string = and_(*query_string)
filtered_entries = source_database.search(query_string)
with disable_undo(source_database):
with disable_undo(destination_database):
source_database.remove_many(filtered_entries)
source_database.commit()
source_database.session.commit()
source_database.session.close()
destination_database.add_many(filtered_entries)
destination_database.commit()
return source_database, destination_database
def split_database(source_database, destination_database, *query_string):
"""
Queries the source database with the query string, and moves the
matched entries to the destination database. When this function is
called, the `~sunpy.database.Database.undo` feature is disabled for both databases.
Parameters
----------
source_database : `~sunpy.database.database.Database`
A SunPy `~Database` object. This is the database on which the queries
will be made.
destination_database : `~sunpy.database.database.Database`
A SunPy `~Database` object. This is the database to which the matched
entries will be moved.
query_string : `list`
A variable number of attributes that are chained together via the
boolean AND operator. The | operator may be used between attributes
to express the boolean OR operator.
Examples
--------
The function call in the following example moves those entries from
database1 to database2 which have `~sunpy.net.attrs.Instrument` = 'AIA' or
'ERNE'.
>>> from sunpy.database import Database, split_database
>>> from sunpy.database.tables import display_entries
>>> from sunpy.net import vso, attrs as a
>>> database1 = Database('sqlite:///:memory:')
>>> database2 = Database('sqlite:///:memory:')
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.search(a.Time('2011-05-08', '2011-05-08 00:00:05'), response_format="legacy") # doctest: +REMOTE_DATA
>>> database1.add_from_vso_query_result(qr) # doctest: +REMOTE_DATA
>>> database1, database2 = split_database(database1, database2,
... a.Instrument.aia | a.Instrument.erne) # doctest: +REMOTE_DATA
"""
query_string = and_(*query_string)
filtered_entries = source_database.search(query_string)
with disable_undo(source_database):
with disable_undo(destination_database):
source_database.remove_many(filtered_entries)
source_database.commit()
source_database.session.commit()
source_database.session.close()
destination_database.add_many(filtered_entries)
destination_database.commit()
return source_database, destination_database
def query(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> client.query(
... vso.attrs.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... vso.attrs.Instrument('eit') | vso.attrs.Instrument('aia')) # doctest: +NORMALIZE_WHITESPACE
<Table masked=False length=5>
Start Time [1] End Time [1] Source Instrument Type
string152 string152 string32 string24 string64
------------------- ------------------- -------- ---------- --------
2010-01-01 00:00:08 2010-01-01 00:00:20 SOHO EIT FULLDISK
2010-01-01 00:12:08 2010-01-01 00:12:20 SOHO EIT FULLDISK
2010-01-01 00:24:10 2010-01-01 00:24:22 SOHO EIT FULLDISK
2010-01-01 00:36:08 2010-01-01 00:36:20 SOHO EIT FULLDISK
2010-01-01 00:48:09 2010-01-01 00:48:21 SOHO EIT FULLDISK
Returns
-------
out : :py:class:`QueryResult` (enhanced list) of matched items. Return
value of same type as the one of :py:meth:`VSOClient.query`.
"""
query = and_(*query)
responses = []
for block in walker.create(query, self.api):
try:
responses.append(
self.api.service.Query(
self.make('QueryRequest', block=block)
)
)
except TypeNotFound:
pass
except Exception as ex:
response = QueryResponse.create(self.merge(responses))
response.add_error(ex)
return QueryResponse.create(self.merge(responses))
def query(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> client.query(
... vso.attrs.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... vso.attrs.Instrument('eit') | vso.attrs.Instrument('aia')) # doctest: +NORMALIZE_WHITESPACE
<Table masked=False length=5>
Start Time [1] End Time [1] Source Instrument Type
string152 string152 string32 string24 string64
------------------- ------------------- -------- ---------- --------
2010-01-01 00:00:08 2010-01-01 00:00:20 SOHO EIT FULLDISK
2010-01-01 00:12:08 2010-01-01 00:12:20 SOHO EIT FULLDISK
2010-01-01 00:24:10 2010-01-01 00:24:22 SOHO EIT FULLDISK
2010-01-01 00:36:08 2010-01-01 00:36:20 SOHO EIT FULLDISK
2010-01-01 00:48:09 2010-01-01 00:48:21 SOHO EIT FULLDISK
Returns
-------
out : :py:class:`QueryResult` (enhanced list) of matched items. Return
value of same type as the one of :py:meth:`VSOClient.query`.
"""
query = and_(*query)
responses = []
for block in walker.create(query, self.api):
try:
responses.append(
self.api.service.Query(
self.make('QueryRequest', block=block)
)
)
except TypeNotFound:
pass
except Exception as ex:
response = QueryResponse.create(self.merge(responses))
response.add_error(ex)
return QueryResponse.create(self.merge(responses))
def search(self, *query):
"""
Query for data in form of multiple parameters.
Examples
--------
Query for LYRA timeseries data for the time range ('2012/3/4','2012/3/6')
>>> from sunpy.net import Fido, attrs as a
>>> import astropy.units as u
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'), a.Instrument('lyra')) # doctest: +REMOTE_DATA
Query for data from Nobeyama Radioheliograph and RHESSI
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
... (a.Instrument('norh') & a.Wavelength(17*u.GHz)) | a.Instrument('rhessi')) # doctest: +REMOTE_DATA
Query for 304 Angstrom SDO AIA data with a cadence of 10 minutes
>>> import astropy.units as u
>>> from sunpy.net import Fido, attrs as a
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
... a.Instrument('AIA'),
... a.Wavelength(304*u.angstrom, 304*u.angstrom),
... a.Sample(10*u.minute)) # doctest: +REMOTE_DATA
Parameters
----------
query : `sunpy.net.vso.attrs`, `sunpy.net.jsoc.attrs`
A query consisting of multiple parameters which define the
requested data. The query is specified using attributes from the
VSO and the JSOC. The query can mix attributes from the VSO and
the JSOC.
Returns
-------
`sunpy.net.fido_factory.UnifiedResponse`
Container of responses returned by clients servicing query.
Notes
-----
The conjunction 'and' transforms query into disjunctive normal form
ie. query is now of form A & B or ((A & B) | (C & D))
This helps in modularising query into parts and handling each of the
parts individually.
""" # noqa
query = attr.and_(*query)
return UnifiedResponse(query_walker.create(query, self))
def search(self, *query):
"""
Query for data in form of multiple parameters.
Examples
--------
Query for LYRALightCurve data for the time range ('2012/3/4','2012/3/6')
>>> from sunpy.net import Fido, attrs as a
>>> import astropy.units as u
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'), a.Instrument('lyra')) # doctest: +REMOTE_DATA
Query for data from Nobeyama Radioheliograph and RHESSI
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
... (a.Instrument('norh') & a.Wavelength(17*u.GHz)) | a.Instrument('rhessi')) # doctest: +REMOTE_DATA
Query for 304 Angstrom SDO AIA data with a cadence of 10 minutes
>>> import astropy.units as u
>>> from sunpy.net import Fido, attrs as a
>>> unifresp = Fido.search(a.Time('2012/3/4', '2012/3/6'),
... a.Instrument('AIA'),
... a.Wavelength(304*u.angstrom, 304*u.angstrom),
... a.vso.Sample(10*u.minute)) # doctest: +REMOTE_DATA
Parameters
----------
query : `sunpy.net.vso.attrs`, `sunpy.net.jsoc.attrs`
A query consisting of multiple parameters which define the
requested data. The query is specified using attributes from the
VSO and the JSOC. The query can mix attributes from the VSO and
the JSOC.
Returns
-------
`sunpy.net.fido_factory.UnifiedResponse`
Container of responses returned by clients servicing query.
Notes
-----
The conjunction 'and' transforms query into disjunctive normal form
ie. query is now of form A & B or ((A & B) | (C & D))
This helps in modularising query into parts and handling each of the
parts individually.
""" # noqa
query = attr.and_(*query)
return UnifiedResponse(query_walker.create(query, self))
def search(self, *args) -> DKISTQueryResponseTable:
"""
Search for datasets provided by the DKIST data centre.
"""
query = attr.and_(*args)
queries = walker.create(query)
results = []
for url_parameters in queries:
query_string = urllib.parse.urlencode(url_parameters)
full_url = f"{self._BASE_SEARCH_URL}?{query_string}"
data = urllib.request.urlopen(full_url)
data = json.loads(data.read())
results += data["searchResults"]
return DKISTQueryResponseTable.from_results(results, client=self)
def query(self, *query):
""" Retrieve information about records matching the criteria
given in the query expression. If multiple arguments are passed,
they are connected with AND. """
query = attr.and_(*query)
data = attrs.walker.create(query, {})
ndata = []
for elem in data:
new = self.default.copy()
new.update(elem)
ndata.append(new)
if len(ndata) == 1:
return self._download(ndata[0])
else:
return self._merge(self._download(data) for data in ndata)
def query(self, *query):
""" Retrieve information about records matching the criteria
given in the query expression. If multiple arguments are passed,
they are connected with AND. """
query = attr.and_(*query)
data = attrs.walker.create(query, {})
ndata = []
for elem in data:
new = self.default.copy()
new.update(elem)
ndata.append(new)
if len(ndata) == 1:
return self._download(ndata[0])
else:
return self._merge(self._download(data) for data in ndata)
def fetch(self, *query, **kwargs):
"""fetch(*query[, path])
Check if the query has already been used to collect new data using the
:meth:`sunpy.database.Database.download` method. If yes, query the
database using the method :meth:`sunpy.database.Database.query` and
return the result. Otherwise, call
:meth:`sunpy.database.Database.download` and return the result.
"""
if not query:
raise TypeError('at least one attribute required')
dump = serialize.dump_query(and_(*query))
(dump_exists,), = self.session.query(
exists().where(tables.JSONDump.dump == tables.JSONDump(dump).dump))
if dump_exists:
return self.query(*query)
return self.download(*query, **kwargs)
def search(self, *query):
""" Retrieves information about HEK records matching the criteria
given in the query expression. If multiple arguments are passed,
they are connected with AND. The result of a query is a list of
unique HEK Response objects that fulfill the criteria."""
query = attr.and_(*query)
data = attrs.walker.create(query, {})
ndata = []
for elem in data:
new = self.default.copy()
new.update(elem)
ndata.append(new)
if len(ndata) == 1:
return self._download(ndata[0])
else:
return self._merge(self._download(data) for data in ndata)
def fetch(self, *query, **kwargs):
"""fetch(*query[, path])
Check if the query has already been used to collect new data using the
:meth:`sunpy.database.Database.download` method. If yes, query the
database using the method :meth:`sunpy.database.Database.query` and
return the result. Otherwise, call
:meth:`sunpy.database.Database.download` and return the result.
"""
if not query:
raise TypeError('at least one attribute required')
dump = serialize.dump_query(and_(*query))
(dump_exists, ), = self.session.query(
exists().where(tables.JSONDump.dump == tables.JSONDump(dump).dump))
if dump_exists:
return self.query(*query)
return self.download(*query, **kwargs)
def search(self, *query):
""" Retrieves information about HEK records matching the criteria
given in the query expression. If multiple arguments are passed,
they are connected with AND. The result of a query is a list of
unique HEK Response objects that fulfill the criteria."""
query = attr.and_(*query)
data = attrs.walker.create(query, {})
ndata = []
for elem in data:
new = self.default.copy()
new.update(elem)
ndata.append(new)
if len(ndata) == 1:
return self._download(ndata[0])
else:
return self._merge(self._download(data) for data in ndata)
def search(self, *query, **kwargs):
"""
Build a JSOC query and submit it to JSOC for processing.
Takes a variable number of :mod:`sunpy.net.jsoc.attrs` as parameters,
which are chained together using the AND (`&`) operator.
Complex queries to be easily formed using logical operators such as
`&` and `|`, in the same way as the VSO client.
Examples
--------
Request all AIA 304 image data between 2010-01-01T00:00 and
2010-01-01T01:00 in rice compressed form.
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient()
>>> response = client.search(a.Time('2010-01-01T00:00:00', '2010-01-01T01:00:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(304*u.AA),
... a.jsoc.Compression('rice'), a.jsoc.Segment('image'))
Returns
-------
results : JSOCResults object
A collection of records that the query returns.
"""
return_results = JSOCResponse()
query = and_(*query)
blocks = []
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
blocks.append(iargs)
return_results.append(self._lookup_records(iargs))
return_results.query_args = blocks
return return_results
def query(self, *query, **kwargs):
"""
Build a JSOC query and submit it to JSOC for processing.
Takes a variable number of :mod:`sunpy.net.jsoc.attrs` as parameters,
which are chained together using the AND (`&`) operator.
Complex queries to be easily formed using logical operators such as
`&` and `|`, in the same way as the VSO client.
Examples
--------
Request all AIA 304 image data between 2010-01-01T00:00 and
2010-01-01T01:00 in rice compressed form.
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient()
>>> response = client.query(a.Time('2010-01-01T00:00:00', '2010-01-01T01:00:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(304*u.AA),
... a.jsoc.Compression('rice'), a.jsoc.Segment('image'))
Returns
-------
results : JSOCResults object
A collection of records that the query returns.
"""
return_results = JSOCResponse()
query = and_(*query)
blocks = []
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
blocks.append(iargs)
return_results.append(self._lookup_records(iargs))
return_results.query_args = blocks
return return_results
def search(self, *args, **kwargs):
"""
Retrieves information about HEK records matching the criteria
given in the query expression. If multiple arguments are passed,
they are connected with AND. The result of a query is a list of
unique HEK Response objects that fulfill the criteria.
Examples
-------
>>> from sunpy.net import attrs as a, Fido
>>> timerange = a.Time('2011/08/09 07:23:56', '2011/08/09 12:40:29')
>>> res = Fido.search(timerange, a.hek.FL, a.hek.FRM.Name == "SWPC") # doctest: +REMOTE_DATA
>>> res #doctest: +SKIP
<sunpy.net.fido_factory.UnifiedResponse object at ...>
Results from 1 Provider:
<BLANKLINE>
2 Results from the HEKClient:
SOL_standard active ... skel_startc2 sum_overlap_scores
------------------------------ ------ ... ------------ ------------------
SOL2011-08-09T07:19:00L227C090 true ... None 0
SOL2011-08-09T07:48:00L296C073 true ... None 0
<BLANKLINE>
<BLANKLINE>
"""
query = attr.and_(*args)
data = attrs.walker.create(query, {})
ndata = []
for elem in data:
new = self.default.copy()
new.update(elem)
ndata.append(new)
if len(ndata) == 1:
return HEKTable(self._download(ndata[0]), client=self)
else:
return HEKTable(self._merge(
self._download(data) for data in ndata),
client=self)
def test_and_nesting():
a = attr.and_(attrs.Level(0),
attr.AttrAnd((attrs.Instrument('EVE'),
attrs.Time("2012/1/1", "2012/01/02"))))
# Test that the nesting has been removed.
assert len(a.attrs) == 3
def search(self, *query, **kwargs):
"""
Build a JSOC query and submit it to JSOC for processing.
Takes a variable number of `~sunpy.net.jsoc.attrs` as parameters,
which are chained together using the AND (`&`) operator.
Complex queries to be easily formed using logical operators such as
`&` and `|`, in the same way as the VSO client.
Parameters
----------
query : a variable number of `~sunpy.net.jsoc.attrs`
as parameters, which are chained together using
the ``AND`` (``&``) operator.
Returns
-------
response : `~sunpy.net.jsoc.jsoc.JSOCResponse` object
A collection of records that the query returns.
Examples
--------
*Example 1*
Request all AIA 304 image data between 2014-01-01T00:00 and
2014-01-01T01:00::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2017-09-06T12:00:00', '2017-09-06T12:02:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(304*u.AA),
... a.jsoc.Segment('image')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT
-------------------- -------- -------- -------- -------
2017-09-06T11:59:59Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:11Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:23Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:35Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:47Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:59Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:11Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:23Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:35Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:47Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:59Z SDO/AIA AIA_4 304 2194
*Example 2*
Request keyword data of ``hmi.v_45s`` for certain specific keywords only::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2014-01-01T00:00:00', '2014-01-01T00:10:00'),
... a.jsoc.Series('hmi.v_45s'),
... a.jsoc.Keys('T_REC, DATAMEAN, OBS_VR')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC DATAMEAN OBS_VR
----------------------- ------------------ ------------------
2014.01.01_00:00:45_TAI 1906.518188 1911.202614
2014.01.01_00:01:30_TAI 1908.876221 1913.945512
2014.01.01_00:02:15_TAI 1911.7771 1916.6679989999998
2014.01.01_00:03:00_TAI 1913.422485 1919.3699239999999
2014.01.01_00:03:45_TAI 1916.500488 1922.050862
2014.01.01_00:04:30_TAI 1920.414795 1924.7110050000001
2014.01.01_00:05:15_TAI 1922.636963 1927.35015
2014.01.01_00:06:00_TAI 1924.6973879999998 1929.968523
2014.01.01_00:06:45_TAI 1927.758301 1932.5664510000001
2014.01.01_00:07:30_TAI 1929.646118 1935.14288
2014.01.01_00:08:15_TAI 1932.097046 1937.698521
2014.01.01_00:09:00_TAI 1935.7286379999998 1940.23353
2014.01.01_00:09:45_TAI 1937.754028 1942.747605
2014.01.01_00:10:30_TAI 1940.1462399999998 1945.241147
*Example 3*
Request data of ``aia.lev1_euv_12s`` on the basis of PrimeKeys other than ``T_REC``::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2014-01-01T00:00:00', '2014-01-01T00:01:00'),
... a.jsoc.Series('aia.lev1_euv_12s'),
... a.jsoc.PrimeKey('WAVELNTH','171')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT
-------------------- -------- -------- -------- -------
2014-01-01T00:00:01Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:13Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:25Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:37Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:49Z SDO/AIA AIA_3 171 2145
2014-01-01T00:01:01Z SDO/AIA AIA_3 171 2145
"""
return_results = JSOCResponse()
query = and_(*query)
blocks = []
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
blocks.append(iargs)
return_results.append(self._lookup_records(iargs))
return_results.query_args = blocks
return return_results
def vso_all(self):
return attr.and_(self.vso_time, self.vso_instrument)
def vso_all(self):
return attr.and_(self.vso_time, self.vso_instrument)
def search(self, *query, **kwargs):
"""
search(*query[, sortby])
Send the given query to the database and return a list of
database entries that satisfy all of the given attributes.
Apart from the attributes supported by the VSO interface, the following
attributes are supported:
- :class:`sunpy.database.attrs.Starred`
- :class:`sunpy.database.attrs.Tag`
- :class:`sunpy.database.attrs.Path`
- :class:`sunpy.database.attrs.DownloadTime`
- :class:`sunpy.database.attrs.FitsHeaderEntry`
An important difference to the VSO attributes is that these attributes
may also be used in negated form using the tilde ~ operator.
Parameters
----------
query : `list`
A variable number of attributes that are chained together via the
boolean AND operator. The | operator may be used between attributes
to express the boolean OR operator.
sortby : `str`, optional
The column by which to sort the returned entries. The default is to
sort by the start of the observation. See the attributes of
:class:`sunpy.database.tables.DatabaseEntry` for a list of all
possible values.
Returns
-------
table : `list`
List of `sunpy.database.tables.DatabaseEntry` objects that
satisfy all of the given attributes.
Raises
------
TypeError
if no attribute is given or if some keyword argument other than
'sortby' is given.
Examples
--------
The query in the following example searches for all non-starred entries
with the tag 'foo' or 'bar' (or both).
>>> database.search(~attrs.Starred(), attrs.Tag('foo') | attrs.Tag('bar')) # doctest: +SKIP
"""
if not query:
raise TypeError('at least one attribute required')
sortby = kwargs.pop('sortby', 'observation_time_start')
if kwargs:
k, v = kwargs.popitem()
raise TypeError(f'unexpected keyword argument {k!r}')
db_entries = walker.create(and_(*query), self.session)
# If any of the DatabaseEntry-s lack the sorting attribute, the
# sorting key should fall back to 'id', orherwise it fails with
# TypeError on py3
if any([getattr(entry, sortby) is None for entry in db_entries]):
sortby = 'id'
return sorted(db_entries, key=operator.attrgetter(sortby))
def jsoc_query(self, *query, **kwargs):
"""
Build a JSOC query and submit it to JSOC for processing.
Parameters
----------
start_time: datetime, astropy.time or string
The time in UTC (or astropy.time object) specifying the start time.
end_time: datetime, astropy.time or string
The time in UTC (or astropy.time object) specifying the end time.
series: string
A string representing the JSOC data series to download e.g. 'hmi.M_45s'
notify: string
An email address to get a notification to when JSOC has staged your request
protocol: string
The type of download to request one of ("FITS", "JPEG", "MPG", "MP4", or "as-is").
Only FITS is supported, the others will require extra keywords.
compression: string
'rice' or None, download FITS files with RICE compression.
kwargs: dict
Extra keywords to put into the POST payload to JSOC.
Returns
-------
requestIDs: list
A list of the requestIDs generated from your query
"""
# A little (hidden) debug feature
return_resp = kwargs.pop('return_resp', False)
return_response = []
return_reqid = []
query = and_(*query)
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
# Do a multi-request for each query block
responses = self._multi_request(**iargs)
for i, response in enumerate(responses):
#TODD: catch non 200 return
if response.json()['status'] != 2:
warnings.warn(
Warning("Query {0} retuned status {1} with error {2}".format(i,
response.json()['status'],
response.json()['error'])))
responses.pop(i)
#Extract the IDs from the JSON
requestIDs = [response.json()['requestid'] for response in responses]
return_reqid.extend(requestIDs)
return_response.extend(responses)
if return_resp:
return return_response
return return_reqid
def _create_or(walker, query, factory):
qblocks = []
for attrblock in query.attrs:
qblocks.extend(walker.create(attr.and_(attrblock), factory))
return qblocks
def _create_or(walker, query, factory):
qblocks = []
for attrblock in query.attrs:
qblocks.extend(walker.create(attr.and_(attrblock), factory))
return qblocks
def filter_queries(queries):
return attr.and_(queries) not in queries
def search(self, *query, **kwargs):
"""
Build a JSOC query and submit it to JSOC for processing.
Takes a variable number of `~sunpy.net.jsoc.attrs` as parameters,
which are chained together using the AND (`&`) operator.
Complex queries to be easily formed using logical operators such as
`&` and `|`, in the same way as the VSO client.
Parameters
----------
query : a variable number of `~sunpy.net.jsoc.attrs`
as parameters, which are chained together using
the ``AND`` (``&``) operator.
Returns
-------
response : `~sunpy.net.jsoc.jsoc.JSOCResponse` object
A collection of records that the query returns.
Examples
--------
*Example 1*
Request all AIA 304 image data between 2014-01-01T00:00 and
2014-01-01T01:00::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2017-09-06T12:00:00', '2017-09-06T12:02:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.jsoc.Wavelength(304*u.AA),
... a.jsoc.Segment('image')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT
-------------------- -------- -------- -------- -------
2017-09-06T11:59:59Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:11Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:23Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:35Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:47Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:59Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:11Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:23Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:35Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:47Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:59Z SDO/AIA AIA_4 304 2194
*Example 2*
Request keyword data of ``hmi.v_45s`` for certain specific keywords only::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2014-01-01T00:00:00', '2014-01-01T00:10:00'),
... a.jsoc.Series('hmi.v_45s'),
... a.jsoc.Keys('T_REC, DATAMEAN, OBS_VR')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC DATAMEAN OBS_VR
----------------------- ------------------ ------------------
2014.01.01_00:00:45_TAI 1906.518188 1911.202614
2014.01.01_00:01:30_TAI 1908.876221 1913.945512
2014.01.01_00:02:15_TAI 1911.7771 1916.6679989999998
2014.01.01_00:03:00_TAI 1913.422485 1919.3699239999999
2014.01.01_00:03:45_TAI 1916.500488 1922.050862
2014.01.01_00:04:30_TAI 1920.414795 1924.7110050000001
2014.01.01_00:05:15_TAI 1922.636963 1927.35015
2014.01.01_00:06:00_TAI 1924.6973879999998 1929.968523
2014.01.01_00:06:45_TAI 1927.758301 1932.5664510000001
2014.01.01_00:07:30_TAI 1929.646118 1935.14288
2014.01.01_00:08:15_TAI 1932.097046 1937.698521
2014.01.01_00:09:00_TAI 1935.7286379999998 1940.23353
2014.01.01_00:09:45_TAI 1937.754028 1942.747605
2014.01.01_00:10:30_TAI 1940.1462399999998 1945.241147
*Example 3*
Request data of ``aia.lev1_euv_12s`` on the basis of PrimeKeys other than ``T_REC``::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2014-01-01T00:00:00', '2014-01-01T00:01:00'),
... a.jsoc.Series('aia.lev1_euv_12s'),
... a.jsoc.PrimeKey('WAVELNTH','171')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT
-------------------- -------- -------- -------- -------
2014-01-01T00:00:01Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:13Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:25Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:37Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:49Z SDO/AIA AIA_3 171 2145
2014-01-01T00:01:01Z SDO/AIA AIA_3 171 2145
"""
return_results = JSOCResponse()
query = and_(*query)
blocks = []
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
blocks.append(iargs)
return_results.append(self._lookup_records(iargs))
return_results.query_args = blocks
return return_results
def search(self, *query, response_format=None):
"""
Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
Parameters
----------
response_format: {``"legacy"``, ``"table"``}, optional
The response format from the search, this can be either
``"legacy"`` to return a list-like object of the zeep responses, or
``"table"`` to return the responses in a subclass of
`~astropy.table.QTable`.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso, attrs as a
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> client.search(
... a.Time(datetime(2010, 1, 1), datetime(2010, 1, 1, 1)),
... a.Instrument.eit | a.Instrument.aia,
... response_format="table") # doctest: +REMOTE_DATA
<sunpy.net.vso.table_response.VSOQueryResponseTable object at ...>
Start Time End Time Source ... Extent Type Size
... Mibyte
----------------------- ----------------------- ------ ... ----------- -------
2010-01-01 00:00:08.000 2010-01-01 00:00:20.000 SOHO ... FULLDISK 2.01074
2010-01-01 00:12:08.000 2010-01-01 00:12:20.000 SOHO ... FULLDISK 2.01074
2010-01-01 00:24:10.000 2010-01-01 00:24:22.000 SOHO ... FULLDISK 2.01074
2010-01-01 00:36:08.000 2010-01-01 00:36:20.000 SOHO ... FULLDISK 2.01074
2010-01-01 00:48:09.000 2010-01-01 00:48:21.000 SOHO ... FULLDISK 2.01074
Returns
-------
out : `~sunpy.net.vso.table_response.VSOQueryResponseTable`
Matched items. Return value is of same type as the one of
:meth:`VSOClient.search`.
"""
if response_format is None:
response_format = "table"
query = and_(*query)
QueryRequest = self.api.get_type('VSO:QueryRequest')
VSOQueryResponse = self.api.get_type('VSO:QueryResponse')
responses = []
exceptions = []
for block in walker.create(query, self.api):
try:
query_response = self.api.service.Query(
QueryRequest(block=block)
)
for resp in query_response:
if resp["error"]:
warn_user(resp["error"])
responses.append(
VSOQueryResponse(query_response)
)
except Exception as ex:
exceptions.append(ex)
responses = self.merge(responses)
if response_format == "legacy":
response = QueryResponse.create(responses)
else:
response = VSOQueryResponseTable.from_zeep_response(responses, client=self)
for ex in exceptions:
response.add_error(ex)
return response
def filter_queries(queries):
return attr.and_(queries) not in queries
def search(self, *query, **kwargs):
"""
Build a JSOC query and submit it to JSOC for processing.
Takes a variable number of `~sunpy.net.jsoc.attrs` as parameters,
which are chained together using the AND (``&``) operator.
Complex queries to be easily formed using logical operators such as
``&`` and ``|``, in the same way as the VSO client.
Parameters
----------
*query : a variable number of `~sunpy.net.jsoc.attrs`
as parameters, which are chained together using
the ``AND`` (``&``) operator.
Returns
-------
response : `~sunpy.net.jsoc.jsoc.JSOCResponse` object
A collection of records that the query returns.
Examples
--------
*Example 1*
Request all AIA 304 image data between 2014-01-01T00:00 and
2014-01-01T01:00::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2017-09-06T12:00:00', '2017-09-06T12:02:00'),
... a.jsoc.Series('aia.lev1_euv_12s'), a.Wavelength(304*u.AA),
... a.jsoc.Segment('image')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT
-------------------- -------- -------- -------- -------
2017-09-06T11:59:59Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:11Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:23Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:35Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:47Z SDO/AIA AIA_4 304 2194
2017-09-06T12:00:59Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:11Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:23Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:35Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:47Z SDO/AIA AIA_4 304 2194
2017-09-06T12:01:59Z SDO/AIA AIA_4 304 2194
*Example 2*
Request keyword data of ``hmi.v_45s`` and show specific columns only::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2014-01-01T00:00:00', '2014-01-01T00:10:00'),
... a.jsoc.Series('hmi.v_45s')) # doctest: +REMOTE_DATA
>>> print(response.show('T_REC', 'WAVELNTH', 'CAR_ROT')) # doctest: +REMOTE_DATA
T_REC WAVELNTH CAR_ROT
----------------------- -------- -------
2014.01.01_00:00:45_TAI 6173.0 2145
2014.01.01_00:01:30_TAI 6173.0 2145
2014.01.01_00:02:15_TAI 6173.0 2145
2014.01.01_00:03:00_TAI 6173.0 2145
2014.01.01_00:03:45_TAI 6173.0 2145
2014.01.01_00:04:30_TAI 6173.0 2145
2014.01.01_00:05:15_TAI 6173.0 2145
2014.01.01_00:06:00_TAI 6173.0 2145
2014.01.01_00:06:45_TAI 6173.0 2145
2014.01.01_00:07:30_TAI 6173.0 2145
2014.01.01_00:08:15_TAI 6173.0 2145
2014.01.01_00:09:00_TAI 6173.0 2145
2014.01.01_00:09:45_TAI 6173.0 2145
2014.01.01_00:10:30_TAI 6173.0 2145
*Example 3*
Request data of ``aia.lev1_euv_12s`` on the basis of PrimeKeys other than ``T_REC``::
>>> import astropy.units as u
>>> from sunpy.net import jsoc
>>> from sunpy.net import attrs as a
>>> client = jsoc.JSOCClient() # doctest: +REMOTE_DATA
>>> response = client.search(a.Time('2014-01-01T00:00:00', '2014-01-01T00:01:00'),
... a.jsoc.Series('aia.lev1_euv_12s'),
... a.jsoc.PrimeKey('WAVELNTH','171')) # doctest: +REMOTE_DATA
>>> print(response) # doctest: +REMOTE_DATA
T_REC TELESCOP INSTRUME WAVELNTH CAR_ROT
-------------------- -------- -------- -------- -------
2014-01-01T00:00:01Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:13Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:25Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:37Z SDO/AIA AIA_3 171 2145
2014-01-01T00:00:49Z SDO/AIA AIA_3 171 2145
2014-01-01T00:01:01Z SDO/AIA AIA_3 171 2145
"""
return_results = JSOCResponse(client=self)
query = and_(*query)
blocks = []
for block in walker.create(query):
iargs = kwargs.copy()
iargs.update(block)
# Update blocks with deep copy of iargs because in _make_recordset we use .pop() on element from iargs
blocks.append(copy.deepcopy(iargs))
return_results = astropy.table.vstack(
[return_results, self._lookup_records(iargs)])
return_results.query_args = blocks
return_results._original_num_rows = len(return_results)
return return_results