def merge(items, key=(lambda x: x)):
"""
Given sorted lists of iterables, return new iterable that returns
elements of all iterables sorted with respect to key.
"""
state = {}
for item in map(iter, items):
try:
first = next(item)
except StopIteration:
continue
else:
state[item] = (first, key(first))
while state:
for item, (value, tk) in six.iteritems(state):
# Value is biggest.
if all(tk >= k for it, (v, k) in six.iteritems(state)
if it is not item):
yield value
break
try:
n = next(item)
state[item] = (n, key(n))
except StopIteration:
del state[item]
def merge(items, key=(lambda x: x)):
"""
Given sorted lists of iterables, return new iterable that returns
elements of all iterables sorted with respect to key.
"""
state = {}
for item in map(iter, items):
try:
first = next(item)
except StopIteration:
continue
else:
state[item] = (first, key(first))
while state:
for item, (value, tk) in six.iteritems(state):
# Value is biggest.
if all(tk >= k for it, (v, k) in six.iteritems(state)
if it is not item):
yield value
break
try:
n = next(item)
state[item] = (n, key(n))
except StopIteration:
del state[item]
def super(self, *args, **kwargs):
""" Like __call__, only that when you give it super(cls, obj) items,
it will skip the multimethod for cls and use the one for its parent
class. The normal __call__ does not consider this for performance
reasons. """
objs = self.get(*args, **kwargs)
types = tuple(
[
x.__thisclass__.__mro__[1] if isinstance(x, super) else type(x)
for x in objs
]
)
nargs = [
x.__self__ if isinstance(x, super) else x
for x in args
]
for k, elem in six.iteritems(kwargs):
if isinstance(elem, super):
kwargs[k] = elem.__self__
# This code is duplicate for performance reasons.
cached = self.cache.get(types, None)
if cached is not None:
return cached(*nargs, **kwargs)
for signature, fun in reversed(self.methods):
if all(issubclass(ty, sig) for ty, sig in zip(types, signature)):
self.cache[types] = fun
return fun(*nargs, **kwargs)
raise TypeError
def super(self, *args, **kwargs):
""" Like __call__, only that when you give it super(cls, obj) items,
it will skip the multimethod for cls and use the one for its parent
class. The normal __call__ does not consider this for performance
reasons. """
objs = self.get(*args, **kwargs)
types = tuple([
x.__thisclass__.__mro__[1] if isinstance(x, super) else type(x)
for x in objs
])
nargs = [x.__self__ if isinstance(x, super) else x for x in args]
for k, elem in six.iteritems(kwargs):
if isinstance(elem, super):
kwargs[k] = elem.__self__
# This code is duplicate for performance reasons.
cached = self.cache.get(types, None)
if cached is not None:
return cached(*nargs, **kwargs)
for signature, fun in reversed(self.methods):
if all(issubclass(ty, sig) for ty, sig in zip(types, signature)):
self.cache[types] = fun
return fun(*nargs, **kwargs)
raise TypeError
def print_all(key=None):
"""
Provides a table of the complete list of constants.
Parameters
----------
key : Python string or unicode
Key in dictionary `constants`
Returns
-------
table : `astropy.table.Table`
"""
data_rows = []
for key, this_constant in iteritems(constants):
data_rows.append([
key, this_constant.name, this_constant.value,
this_constant.uncertainty,
str(this_constant.unit), this_constant.reference
])
t = Table(rows=data_rows,
names=('key', 'name', 'value', 'uncertainty', 'unit',
'Reference'))
return t
def _freeze(obj):
""" Create hashable representation of result dict. """
if isinstance(obj, dict):
return tuple((k, _freeze(v)) for k, v in iteritems(obj))
if isinstance(obj, list):
return tuple(_freeze(elem) for elem in obj)
return obj
def _extractDateURL(self, url):
"""Extracts the date from a particular url following the pattern"""
# remove the user and passwd from files if there:
url = url.replace("anonymous:[email protected]@", "")
# url_to_list substitutes '.' and '_' for '/' to then create
# a list of all the blocks in times - assuming they are all
# separated with either '.', '_' or '/'
url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
pattern_list = url_to_list(self.pattern)
url_list = url_to_list(url)
time_order = [
'%Y', '%y', '%b', '%B', '%m', '%d', '%j', '%H', '%I', '%M', '%S',
'%e', '%f'
]
final_date = []
final_pattern = []
# Find in directory and filename
for pattern_elem, url_elem in zip(pattern_list, url_list):
time_formats = [x for x in time_order if x in pattern_elem]
if len(time_formats) > 0:
# Find whether there's text that should not be here
toremove = re.split('%.', pattern_elem)
if len(toremove) > 0:
for bit in toremove:
if bit != '':
url_elem = url_elem.replace(bit, '', 1)
pattern_elem = pattern_elem.replace(bit, '', 1)
final_date.append(url_elem)
final_pattern.append(pattern_elem)
for time_bit in time_formats:
time_order.remove(time_bit)
# Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']
# Make all as single strings
date_together = ''.join(final_date)
pattern_together = ''.join(final_pattern)
re_together = pattern_together
for k, v in six.iteritems(TIME_CONVERSIONS):
re_together = re_together.replace(k, v)
# Lists to contain the unique elements of the date and the pattern
final_date = list()
final_pattern = list()
re_together = re_together.replace('[A-Z]', '\\[A-Z]')
for p, r in zip(
pattern_together.split('%')[1:],
re_together.split('\\')[1:]):
if p == 'e':
continue
regexp = r'\{}'.format(r) if not r.startswith('[') else r
pattern = '%{}'.format(p)
date_part = re.search(regexp, date_together)
date_together = date_together[:date_part.start()] + \
date_together[date_part.end():]
if pattern not in final_pattern:
final_pattern.append('%{}'.format(p))
final_date.append(date_part.group())
return datetime.datetime.strptime(' '.join(final_date),
' '.join(final_pattern))
def _freeze(obj):
""" Create hashable representation of result dict. """
if isinstance(obj, dict):
return tuple((k, _freeze(v)) for k, v in iteritems(obj))
if isinstance(obj, list):
return tuple(_freeze(elem) for elem in obj)
return obj
def make_getdatarequest(self, response, methods=None, info=None):
""" Make datarequest with methods from response. """
if methods is None:
methods = self.method_order + ['URL']
return self.create_getdatarequest(
dict((k, [x.fileid for x in v])
for k, v in iteritems(self.by_provider(response))), methods,
info)
def _URL_followsPattern(self, url):
"""Check whether the url provided follows the pattern"""
pattern = self.pattern
for k, v in six.iteritems(TIME_CONVERSIONS):
pattern = pattern.replace(k, v)
matches = re.match(pattern, url)
if matches:
return matches.end() == matches.endpos == len(self.now)
return False
def _extractDateURL(self, url):
"""Extracts the date from a particular url following the pattern"""
# remove the user and passwd from files if there:
url = url.replace("anonymous:[email protected]@", "")
# url_to_list substitutes '.' and '_' for '/' to then create
# a list of all the blocks in times - assuming they are all
# separated with either '.', '_' or '/'
url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
pattern_list = url_to_list(self.pattern)
url_list = url_to_list(url)
time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',
'%H', '%I', '%M', '%S', '%e', '%f']
final_date = []
final_pattern = []
# Find in directory and filename
for pattern_elem, url_elem in zip(pattern_list, url_list):
time_formats = [x for x in time_order if x in pattern_elem]
if len(time_formats) > 0:
# Find whether there's text that should not be here
toremove = re.split('%.', pattern_elem)
if len(toremove) > 0:
for bit in toremove:
if bit != '':
url_elem = url_elem.replace(bit, '', 1)
pattern_elem = pattern_elem.replace(bit, '', 1)
final_date.append(url_elem)
final_pattern.append(pattern_elem)
for time_bit in time_formats:
time_order.remove(time_bit)
# Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']
# Make all as single strings
date_together = ''.join(final_date)
pattern_together = ''.join(final_pattern)
re_together = pattern_together
for k, v in six.iteritems(TIME_CONVERSIONS):
re_together = re_together.replace(k, v)
# Lists to contain the unique elements of the date and the pattern
final_date = list()
final_pattern = list()
re_together = re_together.replace('[A-Z]', '\\[A-Z]')
for p, r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]):
if p == 'e':
continue
regexp = r'\{}'.format(r) if not r.startswith('[') else r
pattern = '%{}'.format(p)
date_part = re.search(regexp, date_together)
date_together = date_together[:date_part.start()] + \
date_together[date_part.end():]
if pattern not in final_pattern:
final_pattern.append('%{}'.format(p))
final_date.append(date_part.group())
return datetime.datetime.strptime(' '.join(final_date),
' '.join(final_pattern))
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in six.iteritems(root.attrs):
typ = key[0]
if typ == 'tag':
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == 'fitsheaderentry':
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(not_(and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion))))
else:
query = query.filter(and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(value_criterion)))
elif typ == 'download time':
start, end, inverted = value
if inverted:
query = query.filter(
~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(
DatabaseEntry.download_time.between(start, end))
elif typ == 'path':
path, inverted = value
if inverted:
# pylint: disable=E711
query = query.filter(or_(
DatabaseEntry.path != path, DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == 'wave':
wavemin, wavemax, waveunit = value
query = query.filter(and_(
DatabaseEntry.wavemin >= wavemin,
DatabaseEntry.wavemax <= wavemax))
elif typ == 'time':
start, end, near = value
query = query.filter(and_(
DatabaseEntry.observation_time_start < end,
DatabaseEntry.observation_time_end > start))
else:
if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(SUPPORTED_NONVSO_ATTRS):
raise NotImplementedError("The attribute {0!r} is not yet supported to query a database.".format(typ))
query = query.filter_by(**{typ: value})
return query.all()
def make_getdatarequest(self, response, methods=None, info=None):
""" Make datarequest with methods from response. """
if methods is None:
methods = self.method_order + ['URL']
return self.create_getdatarequest(
dict((k, [x.fileid for x in v])
for k, v in iteritems(self.by_provider(response))),
methods, info
)
def _apply(wlk, root, api, queryblock):
""" Implementation detail. """
for k, v in iteritems(root.attrs):
lst = k[-1]
rest = k[:-1]
block = queryblock
for elem in rest:
block = block[elem]
block[lst] = v
def _apply(wlk, root, api, queryblock):
""" Implementation detail. """
for k, v in iteritems(root.attrs):
lst = k[-1]
rest = k[:-1]
block = queryblock
for elem in rest:
block = block[elem]
block[lst] = v
def make(self, atype, **kwargs):
""" Create new SOAP object with attributes specified in kwargs.
To assign subattributes, use foo__bar=1 to assign
['foo']['bar'] = 1. """
obj = self.api.factory.create(atype)
for k, v in iteritems(kwargs):
split = k.split('__')
tip = split[-1]
rest = split[:-1]
item = obj
for elem in rest:
item = item[elem]
if isinstance(v, dict):
# Do not throw away type information for dicts.
for k, v in iteritems(v):
item[tip][k] = v
else:
item[tip] = v
return obj
def to_be_removed(self):
"""Returns the key with the lowest times of access and its
corresponding value as a tuple.
"""
min_ = float('inf')
lfu_key = None
for k, v in six.iteritems(self.usage_counter):
if v < min_:
min_ = v
lfu_key = k
return lfu_key, self.get(lfu_key)
def make(self, atype, **kwargs):
""" Create new SOAP object with attributes specified in kwargs.
To assign subattributes, use foo__bar=1 to assign
['foo']['bar'] = 1. """
obj = self.api.factory.create(atype)
for k, v in iteritems(kwargs):
split = k.split('__')
tip = split[-1]
rest = split[:-1]
item = obj
for elem in rest:
item = item[elem]
if isinstance(v, dict):
# Do not throw away type information for dicts.
for k, v in iteritems(v):
item[tip][k] = v
else:
item[tip] = v
return obj
def make(self, atype, **kwargs):
""" Create new SOAP object with attributes specified in kwargs.
To assign subattributes, use foo__bar=1 to assign
['foo']['bar'] = 1. """
obj = self.api.factory.create(atype)
for k, v in iteritems(kwargs):
split = k.split('__')
tip = split[-1]
rest = split[:-1]
item = obj
for elem in rest:
item = item[elem]
def find_time(string, format):
""" Return iterator of occurrences of date formatted with format
in string. Currently supported format codes: """
re_format = format
for key, value in six.iteritems(REGEX):
re_format = re_format.replace(key, value)
matches = re.finditer(re_format, string)
for match in matches:
try:
matchstr = string[slice(*match.span())]
dt = datetime.strptime(matchstr, format)
except ValueError:
continue
else:
yield dt
def create_getdatarequest(self, maps, methods, info=None):
""" Create datarequest from maps mapping data provider to
fileids and methods, """
if info is None:
info = {}
return self.make(
'VSOGetDataRequest',
request__method__methodtype=methods,
request__info=info,
request__datacontainer__datarequestitem=[
self.make('DataRequestItem', provider=k, fileiditem__fileid=[v])
for k, v in iteritems(maps)
]
)
def find_time(string, format):
""" Return iterator of occurrences of date formatted with format
in string. Currently supported format codes: """
re_format = format
for key, value in six.iteritems(REGEX):
re_format = re_format.replace(key, value)
matches = re.finditer(re_format, string)
for match in matches:
try:
matchstr = string[slice(*match.span())]
dt = datetime.strptime(matchstr, format)
except ValueError:
continue
else:
yield dt
def create_getdatarequest(self, maps, methods, info=None):
""" Create datarequest from maps mapping data provider to
fileids and methods, """
if info is None:
info = {}
return self.make('VSOGetDataRequest',
request__method__methodtype=methods,
request__info=info,
request__datacontainer__datarequestitem=[
self.make('DataRequestItem',
provider=k,
fileiditem__fileid=[v])
for k, v in iteritems(maps)
])
def _extractDateURL(self, url):
"""Extracts the date from a particular url following the pattern"""
# url_to_list substitutes '.' and '_' for '/' to then create
# a list of all the blocks in times - assuming they are all
# separated with either '.', '_' or '/'
url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
pattern_list = url_to_list(self.pattern)
url_list = url_to_list(url)
time_order = [
'%Y', '%y', '%b', '%B', '%m', '%d', '%j', '%H', '%I', '%M', '%S'
]
final_date = []
final_pattern = []
# Find in directory and filename
for pattern_elem, url_elem in zip(pattern_list, url_list):
time_formats = [x for x in time_order if x in pattern_elem]
if len(time_formats) > 0:
final_date.append(url_elem)
final_pattern.append(pattern_elem)
for time_bit in time_formats:
time_order.remove(time_bit)
# Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']
# Make all as single strings
date_together = ''.join(final_date)
pattern_together = ''.join(final_pattern)
re_together = pattern_together
for k, v in six.iteritems(TIME_CONVERSIONS):
re_together = re_together.replace(k, v)
# Create new empty lists
final_date = list()
final_pattern = list()
for p, r in zip(
pattern_together.split('%')[1:],
re_together.split('\\')[1:]):
regexp = '\\{}'.format(r)
pattern = '%{}'.format(p)
date_part = re.match(regexp, date_together)
date_together = date_together[:date_part.start()] + \
date_together[date_part.end():]
if pattern not in final_pattern:
final_pattern.append('%{}'.format(p))
final_date.append(date_part.group())
return datetime.datetime.strptime(' '.join(final_date),
' '.join(final_pattern))
def _close(self, callback, args, server):
""" Called after download is done. Activated queued downloads, call callback.
"""
callback(*args)
self.connections[server] -= 1
self.conns -= 1
if self.q[server]:
self._attempt_download(*self.q[server].pop())
else:
for k, v in iteritems(self.q): # pylint: disable=W0612
while v:
if self._attempt_download(*v[0]):
v.popleft()
if self.conns == self.max_total:
return
else:
break
def _extractDateURL(self, url):
"""Extracts the date from a particular url following the pattern"""
# url_to_list substitutes '.' and '_' for '/' to then create
# a list of all the blocks in times - assuming they are all
# separated with either '.', '_' or '/'
url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/')
pattern_list = url_to_list(self.pattern)
url_list = url_to_list(url)
time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j',
'%H', '%I', '%M', '%S']
final_date = []
final_pattern = []
# Find in directory and filename
for pattern_elem, url_elem in zip(pattern_list, url_list):
time_formats = [x for x in time_order if x in pattern_elem]
if len(time_formats) > 0:
final_date.append(url_elem)
final_pattern.append(pattern_elem)
for time_bit in time_formats:
time_order.remove(time_bit)
# Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d']
# Make all as single strings
date_together = ''.join(final_date)
pattern_together = ''.join(final_pattern)
re_together = pattern_together
for k, v in six.iteritems(TIME_CONVERSIONS):
re_together = re_together.replace(k, v)
# Create new empty lists
final_date = list()
final_pattern = list()
for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]):
regexp = '\\{}'.format(r)
pattern = '%{}'.format(p)
date_part = re.match(regexp, date_together)
date_together = date_together[:date_part.start()] + \
date_together[date_part.end():]
if pattern not in final_pattern:
final_pattern.append('%{}'.format(p))
final_date.append(date_part.group())
return datetime.datetime.strptime(' '.join(final_date),
' '.join(final_pattern))
def _close(self, callback, args, server):
""" Called after download is done. Activated queued downloads, call callback.
"""
callback(*args)
self.connections[server] -= 1
self.conns -= 1
if self.q[server]:
self._attempt_download(*self.q[server].pop())
else:
for k, v in iteritems(self.q): # pylint: disable=W0612
while v:
if self._attempt_download(*v[0]):
v.popleft()
if self.conns == self.max_total:
return
else:
break
def print_all(key=None):
"""
Provides a table of the complete list of constants.
Parameters
----------
key : Python string or unicode
Key in dictionary `constants`
Returns
-------
table : `astropy.table.Table`
"""
data_rows = []
for key, this_constant in iteritems(constants):
data_rows.append([key, this_constant.name, this_constant.value, this_constant.uncertainty,
str(this_constant.unit), this_constant.reference])
t = Table(rows=data_rows, names=('key', 'name', 'value', 'uncertainty', 'unit', 'Reference'))
return t
def _regex_parse_time(inp, format):
# Parser for finding out the minute value so we can adjust the string
# from 24:00:00 to 00:00:00 the next day because strptime does not
# understand the former.
for key, value in six.iteritems(REGEX):
format = format.replace(key, value)
match = re.match(format, inp)
if match is None:
return None, None
try:
hour = match.group("hour")
except IndexError:
return inp, timedelta(days=0)
if match.group("hour") == "24":
if not all(_n_or_eq(_group_or_none(match, g, int), 00)
for g in ["minute", "second", "microsecond"]
):
raise ValueError
from_, to = match.span("hour")
return inp[:from_] + "00" + inp[to:], timedelta(days=1)
return inp, timedelta(days=0)
def create_getdatarequest(self, maps, methods, info=None):
""" Create datarequest from maps mapping data provider to
fileids and methods, """
if info is None:
info = {}
# For the JSOC provider we need to make a DataRequestItem for each
# series, not just one for the whole provider.
# Remove JSOC provider items from the map
jsoc = maps.pop('JSOC', [])
# Make DRIs for everything that's not JSOC one per provider
dris = [self.make('DataRequestItem', provider=k, fileiditem__fileid=[v])
for k, v in iteritems(maps)]
def series_func(x):
""" Extract the series from the fileid. """
return x.split(':')[0]
# Sort the JSOC fileids by series
# This is a precursor to groupby as recommended by the groupby docs
series_sorted = sorted(jsoc, key=series_func)
# Iterate over the series and make a DRI for each.
# groupby creates an iterator based on a key function, in this case
# based on the series (the part before the first ':')
for series, fileids in itertools.groupby(series_sorted, key=series_func):
dris.append(self.make('DataRequestItem',
provider='JSOC',
fileiditem__fileid=[list(fileids)]))
return self.make(
'VSOGetDataRequest',
request__method__methodtype=methods,
request__info=info,
request__datacontainer__datarequestitem=dris
)
def entries_from_file(file, default_waveunit=None):
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> from sunpy.database.tables import entries_from_file
>>> import sunpy.data
>>> sunpy.data.download_sample_data(overwrite=False) # doctest: +SKIP
>>> import sunpy.data.sample
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE))
>>> len(entries)
1
>>> entry = entries.pop()
>>> entry.instrument
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end
(datetime.datetime(2012, 1, 1, 0, 16, 7, 836000), None)
>>> entry.wavemin, entry.wavemax
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries)
111
"""
headers = fits.get_header(file)
if isinstance(file, (str, six.text_type)):
filename = file
else:
filename = getattr(file, 'name', None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in six.iteritems(header):
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == '':
value = str(value)
elif key == 'KEYCOMMENTS':
for k, v in six.iteritems(value):
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == 'INSTRUME':
entry.instrument = value
elif key == 'WAVELNTH':
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(
nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ('DATE-END', 'DATE_END'):
entry.observation_time_end = parse_time(value)
elif key in ('DATE-OBS', 'DATE_OBS'):
entry.observation_time_start = parse_time(value)
yield entry
from __future__ import absolute_import
from sunpy.sun import constants as con
from astropy.constants import Constant
import pytest
from sunpy.extern.six import iteritems
@pytest.mark.parametrize('this_constant',
[value for key, value in iteritems(con.constants)])
def test_all_constants_are_constants(this_constant):
"""Test that each member of the constants dict is an astropy Constant"""
assert type(this_constant) is Constant
@pytest.mark.parametrize('this_key',
[key for key, value in iteritems(con.constants)])
def test_get_function(this_key):
"""Test that the get function works for all the keys"""
assert type(con.get(this_key)) is Constant
@pytest.mark.parametrize('this_key',
[key for key, value in iteritems(con.constants)])
def test_find_function(this_key):
"""Test that the find function works for all the keys"""
assert len(con.find(this_key)) >= 1
@pytest.mark.parametrize('this_key',
[key for key, value in iteritems(con.constants)])
def test_find_function(this_key):
def __call__(self):
for k, v in six.iteritems(self.kwargs):
# save those values in the dict prev_values that will be changed
# so that they can be recovered
self.prev_values[k] = getattr(self.database_entry, k)
setattr(self.database_entry, k, v)
# pylint: disable=E0102,C0103,W0613
def _create(wlk, root, api):
""" Implementation detail. """
blocks = []
for attr in root.attrs:
blocks.extend(wlk.create(attr, api))
return blocks
# Converters take a type unknown to the walker and convert it into one
# known to it. All of those convert types into ValueAttrs, which are
# handled above by just assigning according to the keys and values of the
# attrs member.
walker.add_converter(Extent)(
lambda x: ValueAttr(
dict((('extent', k), v) for k, v in iteritems(vars(x)))
)
)
walker.add_converter(Time)(
lambda x: ValueAttr({
('time', 'start'): x.start.strftime(TIMEFORMAT),
('time', 'end'): x.end.strftime(TIMEFORMAT) ,
('time', 'near'): (
x.near.strftime(TIMEFORMAT) if x.near is not None else None),
})
)
walker.add_converter(_VSOSimpleAttr)(
lambda x: ValueAttr({(x.__class__.__name__.lower(), ): x.value})
)
def undo(self):
for k, v in six.iteritems(self.prev_values):
setattr(self.database_entry, k, v)
def __hash__(self):
return hash(frozenset(iteritems(vars(self))))
def __hash__(self):
return hash(frozenset(iteritems(self.attrs.iteritems)))
from __future__ import absolute_import
from sunpy.sun import constants as con
from astropy.constants import Constant
import pytest
from sunpy.extern.six import iteritems
@pytest.mark.parametrize('this_constant', [value for key, value in iteritems(con.constants)])
def test_all_constants_are_constants(this_constant):
"""Test that each member of the constants dict is an astropy Constant"""
assert isinstance(this_constant, Constant)
@pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)])
def test_get_function(this_key):
"""Test that the get function works for all the keys"""
assert isinstance(con.get(this_key), Constant)
@pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)])
def test_find_function(this_key):
"""Test that the find function works for all the keys"""
assert len(con.find(this_key)) >= 1
@pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)])
def test_find_function(this_key):
"""Test that the find function works for all the keys"""
assert len(con.find(this_key)) >= 1
def entries_from_file(file,
default_waveunit=None,
time_string_parse_format=None):
"""Use the headers of a FITS file to generate an iterator of
:class:`sunpy.database.tables.DatabaseEntry` instances. Gathered
information will be saved in the attribute `fits_header_entries`. If the
key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute
`instrument`, `wavemin` and `wavemax` or `observation_time_start` is set,
respectively. If the wavelength unit can be read, the values of `wavemin`
and `wavemax` are converted to nm (nanometres). The value of the `file`
parameter is used to set the attribute `path` of each generated database
entry.
Parameters
----------
file : str or file-like object
Either a path pointing to a FITS file or a an opened file-like object.
If an opened file object, its mode must be one of the following rb,
rb+, or ab+.
default_waveunit : str, optional
The wavelength unit that is used for a header if it cannot be
found.
time_string_parse_format : str, optional
Fallback timestamp format which will be passed to
`~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to
automatically read the `date-obs` metadata.
Raises
------
sunpy.database.WaveunitNotFoundError
If `default_waveunit` is not given and the wavelength unit cannot
be found in one of the FITS headers
sunpy.WaveunitNotConvertibleError
If a wavelength unit could be found but cannot be used to create an
instance of the type ``astropy.units.Unit``. This can be the case
for example if a FITS header has the key `WAVEUNIT` with the value
`nonsense`.
Examples
--------
>>> from sunpy.database.tables import entries_from_file
>>> import sunpy.data.sample # doctest: +REMOTE_DATA
>>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA
>>> len(entries) # doctest: +REMOTE_DATA
1
>>> entry = entries.pop() # doctest: +REMOTE_DATA
>>> entry.instrument # doctest: +REMOTE_DATA
'SWAP'
>>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA
(datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None)
>>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA
(17.400000000000002, 17.400000000000002)
>>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA
111
"""
headers = fits.get_header(file)
if isinstance(file, (str, six.text_type)):
filename = file
else:
filename = getattr(file, 'name', None)
for header in headers:
entry = DatabaseEntry(path=filename)
for key, value in six.iteritems(header):
# Yes, it is possible to have an empty key in a FITS file.
# Example: sunpy.data.sample.EIT_195_IMAGE
# Don't ask me why this could be a good idea.
if key == '':
value = str(value)
elif key == 'KEYCOMMENTS':
for k, v in six.iteritems(value):
entry.fits_key_comments.append(FitsKeyComment(k, v))
continue
entry.fits_header_entries.append(FitsHeaderEntry(key, value))
waveunit = fits.extract_waveunit(header)
entry.hdu_index = headers.index(header)
if waveunit is None:
waveunit = default_waveunit
unit = None
if waveunit is not None:
try:
unit = Unit(waveunit)
except ValueError:
raise WaveunitNotConvertibleError(waveunit)
for header_entry in entry.fits_header_entries:
key, value = header_entry.key, header_entry.value
if key == 'INSTRUME':
entry.instrument = value
elif key == 'WAVELNTH':
if unit is None:
raise WaveunitNotFoundError(file)
# use the value of `unit` to convert the wavelength to nm
entry.wavemin = entry.wavemax = unit.to(
nm, value, equivalencies.spectral())
# NOTE: the key DATE-END or DATE_END is not part of the official
# FITS standard, but many FITS files use it in their header
elif key in ('DATE-END', 'DATE_END'):
entry.observation_time_end = parse_time(
value, _time_string_parse_format=time_string_parse_format)
elif key in ('DATE-OBS', 'DATE_OBS'):
entry.observation_time_start = parse_time(
value, _time_string_parse_format=time_string_parse_format)
yield entry
def create_getdatarequest(self, maps, methods, info=None):
""" Create datarequest from maps mapping data provider to
fileids and methods, """
if info is None:
info = {}
if maps.get('JSOC',0):
return self._separate_hmi(maps, methods, info)
else:
return self.make(
'VSOGetDataRequest',
request__method__methodtype=methods,
request__info=info,
request__datacontainer__datarequestitem=[
self.make('DataRequestItem', provider=k, fileiditem__fileid=[v])
for k, v in iteritems(maps)
]
)
def _separate_hmi(self, maps_dict, methods, info):
"""
This function parses the separate hmi data series from the
maps_dict(jsoc) list, pulls any hmi data it finds into separate lists
and deletes the hmi data from the original jsoc list. It then continues
as normal to construct the soap message.
"""
jsoc_list=maps_dict.get('JSOC',0)
if jsoc_list:
def query_legacy(self, tstart=None, tend=None, **kwargs):
"""
Query data from the VSO mocking the IDL API as close as possible.
Either tstart and tend or date_start and date_end or date have
to be supplied.
Parameters
----------
tstart : datetime.datetime
Start of the time-range in which records are searched.
tend : datetime.datetime
Start of the time-range in which records are searched.
date : str
(start date) - (end date)
start_date : datetime
the start date
end_date : datetime
the end date
wave : str
(min) - (max) (unit)
min_wave : str
minimum spectral range
max_wave : str
maximum spectral range
unit_wave : str
spectral range units (Angstrom, GHz, keV)
extent : str
VSO 'extent type' ... (FULLDISK, CORONA, LIMB, etc)
physobj : str
VSO 'physical observable'
provider : str
VSO ID for the data provider (SDAC, NSO, SHA, MSU, etc)
source : str
spacecraft or observatory (SOHO, YOHKOH, BBSO, etc)
synonyms : spacecraft, observatory
instrument : str
instrument ID (EIT, SXI-0, SXT, etc)
synonyms : telescope, inst
detector : str
detector ID (C3, EUVI, COR2, etc.)
layout : str
layout of the data (image, spectrum, time_series, etc.)
level : str
level of the data product (numeric range, see below)
pixels : str
number of pixels (numeric range, see below)
resolution : str
effective resolution (1 = full, 0.5 = 2x2 binned, etc)
numeric range, see below.
pscale : str
pixel scale, in arcseconds (numeric range, see below)
near_time : datetime
return record closest to the time. See below.
sample : int
attempt to return only one record per SAMPLE seconds. See below.
Numeric Ranges:
- May be entered as a string or any numeric type for equality matching
- May be a string of the format '(min) - (max)' for range matching
- May be a string of the form '(operator) (number)' where operator
is one of: lt gt le ge < > <= >=
Examples
--------
Query all data from eit between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso
>>> client = vso.VSOClient() # doctest: +REMOTE_DATA
>>> qr = client.query_legacy(datetime(2010, 1, 1),
... datetime(2010, 1, 1, 1), instrument='eit') # doctest: +REMOTE_DATA
Returns
-------
out : :py:class:`QueryResult` (enhanced list)
Matched items. Return value is of same type as the one of
:py:class:`VSOClient.query`.
"""
sdk = lambda key: lambda value: {key: value}
ALIASES = {
'wave_min': sdk('wave_wavemin'),
'wave_max': sdk('wave_wavemax'),
'wave_type': sdk('wave_wavetype'),
'wave_unit': sdk('wave_waveunit'),
'min_wave': sdk('wave_wavemin'),
'max_wave': sdk('wave_wavemax'),
'type_wave': sdk('wave_wavetype'),
'unit_wave': sdk('wave_waveunit'),
'wave': _parse_waverange,
'inst': sdk('instrument'),
'telescope': sdk('instrument'),
'spacecraft': sdk('source'),
'observatory': sdk('source'),
'start_date': sdk('time_start'),
'end_date': sdk('time_end'),
'start': sdk('time_start'),
'end': sdk('time_end'),
'near_time': sdk('time_near'),
'date': _parse_date,
'layout': sdk('datatype'),
}
if tstart is not None:
kwargs.update({'time_start': tstart})
if tend is not None:
kwargs.update({'time_end': tend})
queryreq = self.api.factory.create('QueryRequest')
for key, value in iteritems(kwargs):
for k, v in iteritems(ALIASES.get(key, sdk(key))(value)):
if k.startswith('time'):
v = parse_time(v).strftime(TIMEFORMAT)
attr = k.split('_')
lst = attr[-1]
rest = attr[:-1]
# pylint: disable=E1103
item = queryreq.block
for elem in rest:
try:
item = item[elem]
except KeyError:
raise ValueError(
"Unexpected argument {key!s}.".format(key=key))
if lst not in item:
raise ValueError(
"Unexpected argument {key!s}.".format(key=key))
if item[lst]:
raise ValueError(
"Got multiple values for {k!s}.".format(k=k))
item[lst] = v
try:
return QueryResponse.create(self.api.service.Query(queryreq))
except TypeNotFound:
return QueryResponse([])
@walker.add_creator(AttrOr)
# pylint: disable=E0102,C0103,W0613
def _create(wlk, root, api):
""" Implementation detail. """
blocks = []
for attr in root.attrs:
blocks.extend(wlk.create(attr, api))
return blocks
# Converters take a type unknown to the walker and convert it into one
# known to it. All of those convert types into ValueAttrs, which are
# handled above by just assigning according to the keys and values of the
# attrs member.
walker.add_converter(Extent)(lambda x: ValueAttr(
dict((('extent', k), v) for k, v in iteritems(vars(x)))))
walker.add_converter(Time)(lambda x: ValueAttr({
('time', 'start'):
x.start.strftime(TIMEFORMAT),
('time', 'end'):
x.end.strftime(TIMEFORMAT),
('time', 'near'): (x.near.strftime(TIMEFORMAT)
if x.near is not None else None),
}))
walker.add_converter(_VSOSimpleAttr)(
lambda x: ValueAttr({(x.__class__.__name__.lower(), ): x.value}))
walker.add_converter(Wavelength)(
lambda x: ValueAttr({
def __call__(self):
for k, v in six.iteritems(self.kwargs):
# save those values in the dict prev_values that will be changed
# so that they can be recovered
self.prev_values[k] = getattr(self.database_entry, k)
setattr(self.database_entry, k, v)
def __hash__(self):
return hash(frozenset(iteritems(self.attrs.iteritems)))
""" Create new SOAP object with attributes specified in kwargs.
To assign subattributes, use foo__bar=1 to assign
['foo']['bar'] = 1. """
obj = self.api.factory.create(atype)
for k, v in iteritems(kwargs):
split = k.split('__')
tip = split[-1]
rest = split[:-1]
item = obj
for elem in rest:
item = item[elem]
if isinstance(v, dict):
# Do not throw away type information for dicts.
for k, v in iteritems(v):
item[tip][k] = v
else:
item[tip] = v
return obj
def search(self, *query):
""" Query data from the VSO with the new API. Takes a variable number
of attributes as parameter, which are chained together using AND.
The new query language allows complex queries to be easily formed.
Examples
--------
Query all data from eit or aia between 2010-01-01T00:00 and
2010-01-01T01:00.
def __hash__(self):
return hash(frozenset(iteritems(vars(self))))
def undo(self):
for k, v in six.iteritems(self.prev_values):
setattr(self.database_entry, k, v)
from __future__ import absolute_import
from sunpy.sun import constants as con
from astropy.constants import Constant
import pytest
from sunpy.extern.six import iteritems
@pytest.mark.parametrize('this_constant', [value for key, value in iteritems(con.constants)])
def test_all_constants_are_constants(this_constant):
"""Test that each member of the constants dict is an astropy Constant"""
assert type(this_constant) is Constant
@pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)])
def test_get_function(this_key):
"""Test that the get function works for all the keys"""
assert type(con.get(this_key)) is Constant
@pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)])
def test_find_function(this_key):
"""Test that the find function works for all the keys"""
assert len(con.find(this_key)) >= 1
@pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)])
def test_find_function(this_key):
"""Test that the find function works for all the keys"""
assert len(con.find(this_key)) >= 1
def iteritems(self): # pragma: no cover
for key, value in six.iteritems(self._dict):
yield key, value
def query_legacy(self, tstart=None, tend=None, **kwargs):
"""
Query data from the VSO mocking the IDL API as close as possible.
Either tstart and tend or date_start and date_end or date have
to be supplied.
Parameters
----------
tstart : datetime.datetime
Start of the time-range in which records are searched.
tend : datetime.datetime
Start of the time-range in which records are searched.
date : str
(start date) - (end date)
start_date : datetime
the start date
end_date : datetime
the end date
wave : str
(min) - (max) (unit)
min_wave : str
minimum spectral range
max_wave : str
maximum spectral range
unit_wave : str
spectral range units (Angstrom, GHz, keV)
extent : str
VSO 'extent type' ... (FULLDISK, CORONA, LIMB, etc)
physobj : str
VSO 'physical observable'
provider : str
VSO ID for the data provider (SDAC, NSO, SHA, MSU, etc)
source : str
spacecraft or observatory (SOHO, YOHKOH, BBSO, etc)
synonyms : spacecraft, observatory
instrument : str
instrument ID (EIT, SXI-0, SXT, etc)
synonyms : telescope, inst
detector : str
detector ID (C3, EUVI, COR2, etc.)
layout : str
layout of the data (image, spectrum, time_series, etc.)
level : str
level of the data product (numeric range, see below)
pixels : str
number of pixels (numeric range, see below)
resolution : str
effective resolution (1 = full, 0.5 = 2x2 binned, etc)
numeric range, see below.
pscale : str
pixel scale, in arcseconds (numeric range, see below)
near_time : datetime
return record closest to the time. See below.
sample : int
attempt to return only one record per SAMPLE seconds. See below.
Numeric Ranges:
- May be entered as a string or any numeric type for equality matching
- May be a string of the format '(min) - (max)' for range matching
- May be a string of the form '(operator) (number)' where operator is one of: lt gt le ge < > <= >=
Examples
--------
Query all data from eit between 2010-01-01T00:00 and
2010-01-01T01:00.
>>> from datetime import datetime
>>> from sunpy.net import vso
>>> client = vso.VSOClient()
>>> qr = client.query_legacy(datetime(2010, 1, 1),
... datetime(2010, 1, 1, 1), instrument='eit')
Returns
-------
out : :py:class:`QueryResult` (enhanced list) of matched items. Return value of same type as the one of :py:class:`VSOClient.query`.
"""
sdk = lambda key: lambda value: {key: value}
ALIASES = {
'wave_min': sdk('wave_wavemin'),
'wave_max': sdk('wave_wavemax'),
'wave_type': sdk('wave_wavetype'),
'wave_unit': sdk('wave_waveunit'),
'min_wave': sdk('wave_wavemin'),
'max_wave': sdk('wave_wavemax'),
'type_wave': sdk('wave_wavetype'),
'unit_wave': sdk('wave_waveunit'),
'wave': _parse_waverange,
'inst': sdk('instrument'),
'telescope': sdk('instrument'),
'spacecraft': sdk('source'),
'observatory': sdk('source'),
'start_date': sdk('time_start'),
'end_date': sdk('time_end'),
'start': sdk('time_start'),
'end': sdk('time_end'),
'near_time': sdk('time_near'),
'date': _parse_date,
'layout': sdk('datatype'),
}
if tstart is not None:
kwargs.update({'time_start': tstart})
if tend is not None:
kwargs.update({'time_end': tend})
queryreq = self.api.factory.create('QueryRequest')
for key, value in iteritems(kwargs):
for k, v in iteritems(ALIASES.get(key, sdk(key))(value)):
if k.startswith('time'):
v = parse_time(v).strftime(TIMEFORMAT)
attr = k.split('_')
lst = attr[-1]
rest = attr[:-1]
# pylint: disable=E1103
item = queryreq.block
for elem in rest:
try:
item = item[elem]
except KeyError:
raise ValueError("Unexpected argument {key!s}.".format(key=key))
if lst not in item:
raise ValueError("Unexpected argument {key!s}.".format(key=key))
if item[lst]:
raise ValueError("Got multiple values for {k!s}.".format(k=k))
item[lst] = v
try:
return QueryResponse.create(self.api.service.Query(queryreq))
except TypeNotFound:
return QueryResponse([])
def _create(wlk, root, session):
query = session.query(DatabaseEntry)
for key, value in six.iteritems(root.attrs):
typ = key[0]
if typ == 'tag':
criterion = TableTag.name.in_([value])
# `key[1]` is here the `inverted` attribute of the tag. That means
# that if it is True, the given tag must not be included in the
# resulting entries.
if key[1]:
query = query.filter(~DatabaseEntry.tags.any(criterion))
else:
query = query.filter(DatabaseEntry.tags.any(criterion))
elif typ == 'fitsheaderentry':
key, val, inverted = value
key_criterion = TableFitsHeaderEntry.key == key
value_criterion = TableFitsHeaderEntry.value == val
if inverted:
query = query.filter(
not_(
and_(
DatabaseEntry.fits_header_entries.any(
key_criterion),
DatabaseEntry.fits_header_entries.any(
value_criterion))))
else:
query = query.filter(
and_(
DatabaseEntry.fits_header_entries.any(key_criterion),
DatabaseEntry.fits_header_entries.any(
value_criterion)))
elif typ == 'download time':
start, end, inverted = value
if inverted:
query = query.filter(
~DatabaseEntry.download_time.between(start, end))
else:
query = query.filter(
DatabaseEntry.download_time.between(start, end))
elif typ == 'path':
path, inverted = value
if inverted:
# pylint: disable=E711
query = query.filter(
or_(DatabaseEntry.path != path,
DatabaseEntry.path == None))
else:
query = query.filter(DatabaseEntry.path == path)
elif typ == 'wave':
wavemin, wavemax, waveunit = value
query = query.filter(
and_(DatabaseEntry.wavemin >= wavemin,
DatabaseEntry.wavemax <= wavemax))
elif typ == 'time':
start, end, near = value
query = query.filter(
and_(DatabaseEntry.observation_time_start < end,
DatabaseEntry.observation_time_end > start))
else:
if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(
SUPPORTED_NONVSO_ATTRS):
raise NotImplementedError(
"The attribute {0!r} is not yet supported to query a database."
.format(typ))
query = query.filter_by(**{typ: value})
return query.all()
