def merge(items, key=(lambda x: x)): """ Given sorted lists of iterables, return new iterable that returns elements of all iterables sorted with respect to key. """ state = {} for item in map(iter, items): try: first = next(item) except StopIteration: continue else: state[item] = (first, key(first)) while state: for item, (value, tk) in six.iteritems(state): # Value is biggest. if all(tk >= k for it, (v, k) in six.iteritems(state) if it is not item): yield value break try: n = next(item) state[item] = (n, key(n)) except StopIteration: del state[item]
def merge(items, key=(lambda x: x)): """ Given sorted lists of iterables, return new iterable that returns elements of all iterables sorted with respect to key. """ state = {} for item in map(iter, items): try: first = next(item) except StopIteration: continue else: state[item] = (first, key(first)) while state: for item, (value, tk) in six.iteritems(state): # Value is biggest. if all(tk >= k for it, (v, k) in six.iteritems(state) if it is not item): yield value break try: n = next(item) state[item] = (n, key(n)) except StopIteration: del state[item]
def super(self, *args, **kwargs): """ Like __call__, only that when you give it super(cls, obj) items, it will skip the multimethod for cls and use the one for its parent class. The normal __call__ does not consider this for performance reasons. """ objs = self.get(*args, **kwargs) types = tuple( [ x.__thisclass__.__mro__[1] if isinstance(x, super) else type(x) for x in objs ] ) nargs = [ x.__self__ if isinstance(x, super) else x for x in args ] for k, elem in six.iteritems(kwargs): if isinstance(elem, super): kwargs[k] = elem.__self__ # This code is duplicate for performance reasons. cached = self.cache.get(types, None) if cached is not None: return cached(*nargs, **kwargs) for signature, fun in reversed(self.methods): if all(issubclass(ty, sig) for ty, sig in zip(types, signature)): self.cache[types] = fun return fun(*nargs, **kwargs) raise TypeError
def super(self, *args, **kwargs): """ Like __call__, only that when you give it super(cls, obj) items, it will skip the multimethod for cls and use the one for its parent class. The normal __call__ does not consider this for performance reasons. """ objs = self.get(*args, **kwargs) types = tuple([ x.__thisclass__.__mro__[1] if isinstance(x, super) else type(x) for x in objs ]) nargs = [x.__self__ if isinstance(x, super) else x for x in args] for k, elem in six.iteritems(kwargs): if isinstance(elem, super): kwargs[k] = elem.__self__ # This code is duplicate for performance reasons. cached = self.cache.get(types, None) if cached is not None: return cached(*nargs, **kwargs) for signature, fun in reversed(self.methods): if all(issubclass(ty, sig) for ty, sig in zip(types, signature)): self.cache[types] = fun return fun(*nargs, **kwargs) raise TypeError
def print_all(key=None): """ Provides a table of the complete list of constants. Parameters ---------- key : Python string or unicode Key in dictionary `constants` Returns ------- table : `astropy.table.Table` """ data_rows = [] for key, this_constant in iteritems(constants): data_rows.append([ key, this_constant.name, this_constant.value, this_constant.uncertainty, str(this_constant.unit), this_constant.reference ]) t = Table(rows=data_rows, names=('key', 'name', 'value', 'uncertainty', 'unit', 'Reference')) return t
def _freeze(obj): """ Create hashable representation of result dict. """ if isinstance(obj, dict): return tuple((k, _freeze(v)) for k, v in iteritems(obj)) if isinstance(obj, list): return tuple(_freeze(elem) for elem in obj) return obj
def _extractDateURL(self, url): """Extracts the date from a particular url following the pattern""" # remove the user and passwd from files if there: url = url.replace("anonymous:[email protected]@", "") # url_to_list substitutes '.' and '_' for '/' to then create # a list of all the blocks in times - assuming they are all # separated with either '.', '_' or '/' url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/') pattern_list = url_to_list(self.pattern) url_list = url_to_list(url) time_order = [ '%Y', '%y', '%b', '%B', '%m', '%d', '%j', '%H', '%I', '%M', '%S', '%e', '%f' ] final_date = [] final_pattern = [] # Find in directory and filename for pattern_elem, url_elem in zip(pattern_list, url_list): time_formats = [x for x in time_order if x in pattern_elem] if len(time_formats) > 0: # Find whether there's text that should not be here toremove = re.split('%.', pattern_elem) if len(toremove) > 0: for bit in toremove: if bit != '': url_elem = url_elem.replace(bit, '', 1) pattern_elem = pattern_elem.replace(bit, '', 1) final_date.append(url_elem) final_pattern.append(pattern_elem) for time_bit in time_formats: time_order.remove(time_bit) # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d'] # Make all as single strings date_together = ''.join(final_date) pattern_together = ''.join(final_pattern) re_together = pattern_together for k, v in six.iteritems(TIME_CONVERSIONS): re_together = re_together.replace(k, v) # Lists to contain the unique elements of the date and the pattern final_date = list() final_pattern = list() re_together = re_together.replace('[A-Z]', '\\[A-Z]') for p, r in zip( pattern_together.split('%')[1:], re_together.split('\\')[1:]): if p == 'e': continue regexp = r'\{}'.format(r) if not r.startswith('[') else r pattern = '%{}'.format(p) date_part = re.search(regexp, date_together) date_together = date_together[:date_part.start()] + \ date_together[date_part.end():] if pattern not in final_pattern: final_pattern.append('%{}'.format(p)) final_date.append(date_part.group()) return datetime.datetime.strptime(' '.join(final_date), ' '.join(final_pattern))
def _freeze(obj): """ Create hashable representation of result dict. """ if isinstance(obj, dict): return tuple((k, _freeze(v)) for k, v in iteritems(obj)) if isinstance(obj, list): return tuple(_freeze(elem) for elem in obj) return obj
def make_getdatarequest(self, response, methods=None, info=None): """ Make datarequest with methods from response. """ if methods is None: methods = self.method_order + ['URL'] return self.create_getdatarequest( dict((k, [x.fileid for x in v]) for k, v in iteritems(self.by_provider(response))), methods, info)
def _URL_followsPattern(self, url): """Check whether the url provided follows the pattern""" pattern = self.pattern for k, v in six.iteritems(TIME_CONVERSIONS): pattern = pattern.replace(k, v) matches = re.match(pattern, url) if matches: return matches.end() == matches.endpos == len(self.now) return False
def _extractDateURL(self, url): """Extracts the date from a particular url following the pattern""" # remove the user and passwd from files if there: url = url.replace("anonymous:[email protected]@", "") # url_to_list substitutes '.' and '_' for '/' to then create # a list of all the blocks in times - assuming they are all # separated with either '.', '_' or '/' url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/') pattern_list = url_to_list(self.pattern) url_list = url_to_list(url) time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j', '%H', '%I', '%M', '%S', '%e', '%f'] final_date = [] final_pattern = [] # Find in directory and filename for pattern_elem, url_elem in zip(pattern_list, url_list): time_formats = [x for x in time_order if x in pattern_elem] if len(time_formats) > 0: # Find whether there's text that should not be here toremove = re.split('%.', pattern_elem) if len(toremove) > 0: for bit in toremove: if bit != '': url_elem = url_elem.replace(bit, '', 1) pattern_elem = pattern_elem.replace(bit, '', 1) final_date.append(url_elem) final_pattern.append(pattern_elem) for time_bit in time_formats: time_order.remove(time_bit) # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d'] # Make all as single strings date_together = ''.join(final_date) pattern_together = ''.join(final_pattern) re_together = pattern_together for k, v in six.iteritems(TIME_CONVERSIONS): re_together = re_together.replace(k, v) # Lists to contain the unique elements of the date and the pattern final_date = list() final_pattern = list() re_together = re_together.replace('[A-Z]', '\\[A-Z]') for p, r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]): if p == 'e': continue regexp = r'\{}'.format(r) if not r.startswith('[') else r pattern = '%{}'.format(p) date_part = re.search(regexp, date_together) date_together = date_together[:date_part.start()] + \ date_together[date_part.end():] if pattern not in final_pattern: final_pattern.append('%{}'.format(p)) final_date.append(date_part.group()) return datetime.datetime.strptime(' '.join(final_date), ' '.join(final_pattern))
def _create(wlk, root, session): query = session.query(DatabaseEntry) for key, value in six.iteritems(root.attrs): typ = key[0] if typ == 'tag': criterion = TableTag.name.in_([value]) # `key[1]` is here the `inverted` attribute of the tag. That means # that if it is True, the given tag must not be included in the # resulting entries. if key[1]: query = query.filter(~DatabaseEntry.tags.any(criterion)) else: query = query.filter(DatabaseEntry.tags.any(criterion)) elif typ == 'fitsheaderentry': key, val, inverted = value key_criterion = TableFitsHeaderEntry.key == key value_criterion = TableFitsHeaderEntry.value == val if inverted: query = query.filter(not_(and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any(value_criterion)))) else: query = query.filter(and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any(value_criterion))) elif typ == 'download time': start, end, inverted = value if inverted: query = query.filter( ~DatabaseEntry.download_time.between(start, end)) else: query = query.filter( DatabaseEntry.download_time.between(start, end)) elif typ == 'path': path, inverted = value if inverted: # pylint: disable=E711 query = query.filter(or_( DatabaseEntry.path != path, DatabaseEntry.path == None)) else: query = query.filter(DatabaseEntry.path == path) elif typ == 'wave': wavemin, wavemax, waveunit = value query = query.filter(and_( DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax)) elif typ == 'time': start, end, near = value query = query.filter(and_( DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start)) else: if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union(SUPPORTED_NONVSO_ATTRS): raise NotImplementedError("The attribute {0!r} is not yet supported to query a database.".format(typ)) query = query.filter_by(**{typ: value}) return query.all()
def make_getdatarequest(self, response, methods=None, info=None): """ Make datarequest with methods from response. """ if methods is None: methods = self.method_order + ['URL'] return self.create_getdatarequest( dict((k, [x.fileid for x in v]) for k, v in iteritems(self.by_provider(response))), methods, info )
def _apply(wlk, root, api, queryblock): """ Implementation detail. """ for k, v in iteritems(root.attrs): lst = k[-1] rest = k[:-1] block = queryblock for elem in rest: block = block[elem] block[lst] = v
def _apply(wlk, root, api, queryblock): """ Implementation detail. """ for k, v in iteritems(root.attrs): lst = k[-1] rest = k[:-1] block = queryblock for elem in rest: block = block[elem] block[lst] = v
def make(self, atype, **kwargs): """ Create new SOAP object with attributes specified in kwargs. To assign subattributes, use foo__bar=1 to assign ['foo']['bar'] = 1. """ obj = self.api.factory.create(atype) for k, v in iteritems(kwargs): split = k.split('__') tip = split[-1] rest = split[:-1] item = obj for elem in rest: item = item[elem] if isinstance(v, dict): # Do not throw away type information for dicts. for k, v in iteritems(v): item[tip][k] = v else: item[tip] = v return obj
def to_be_removed(self): """Returns the key with the lowest times of access and its corresponding value as a tuple. """ min_ = float('inf') lfu_key = None for k, v in six.iteritems(self.usage_counter): if v < min_: min_ = v lfu_key = k return lfu_key, self.get(lfu_key)
def make(self, atype, **kwargs): """ Create new SOAP object with attributes specified in kwargs. To assign subattributes, use foo__bar=1 to assign ['foo']['bar'] = 1. """ obj = self.api.factory.create(atype) for k, v in iteritems(kwargs): split = k.split('__') tip = split[-1] rest = split[:-1] item = obj for elem in rest: item = item[elem] if isinstance(v, dict): # Do not throw away type information for dicts. for k, v in iteritems(v): item[tip][k] = v else: item[tip] = v return obj
def make(self, atype, **kwargs): """ Create new SOAP object with attributes specified in kwargs. To assign subattributes, use foo__bar=1 to assign ['foo']['bar'] = 1. """ obj = self.api.factory.create(atype) for k, v in iteritems(kwargs): split = k.split('__') tip = split[-1] rest = split[:-1] item = obj for elem in rest: item = item[elem]
def find_time(string, format): """ Return iterator of occurrences of date formatted with format in string. Currently supported format codes: """ re_format = format for key, value in six.iteritems(REGEX): re_format = re_format.replace(key, value) matches = re.finditer(re_format, string) for match in matches: try: matchstr = string[slice(*match.span())] dt = datetime.strptime(matchstr, format) except ValueError: continue else: yield dt
def create_getdatarequest(self, maps, methods, info=None): """ Create datarequest from maps mapping data provider to fileids and methods, """ if info is None: info = {} return self.make( 'VSOGetDataRequest', request__method__methodtype=methods, request__info=info, request__datacontainer__datarequestitem=[ self.make('DataRequestItem', provider=k, fileiditem__fileid=[v]) for k, v in iteritems(maps) ] )
def find_time(string, format): """ Return iterator of occurrences of date formatted with format in string. Currently supported format codes: """ re_format = format for key, value in six.iteritems(REGEX): re_format = re_format.replace(key, value) matches = re.finditer(re_format, string) for match in matches: try: matchstr = string[slice(*match.span())] dt = datetime.strptime(matchstr, format) except ValueError: continue else: yield dt
def create_getdatarequest(self, maps, methods, info=None): """ Create datarequest from maps mapping data provider to fileids and methods, """ if info is None: info = {} return self.make('VSOGetDataRequest', request__method__methodtype=methods, request__info=info, request__datacontainer__datarequestitem=[ self.make('DataRequestItem', provider=k, fileiditem__fileid=[v]) for k, v in iteritems(maps) ])
def _extractDateURL(self, url): """Extracts the date from a particular url following the pattern""" # url_to_list substitutes '.' and '_' for '/' to then create # a list of all the blocks in times - assuming they are all # separated with either '.', '_' or '/' url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/') pattern_list = url_to_list(self.pattern) url_list = url_to_list(url) time_order = [ '%Y', '%y', '%b', '%B', '%m', '%d', '%j', '%H', '%I', '%M', '%S' ] final_date = [] final_pattern = [] # Find in directory and filename for pattern_elem, url_elem in zip(pattern_list, url_list): time_formats = [x for x in time_order if x in pattern_elem] if len(time_formats) > 0: final_date.append(url_elem) final_pattern.append(pattern_elem) for time_bit in time_formats: time_order.remove(time_bit) # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d'] # Make all as single strings date_together = ''.join(final_date) pattern_together = ''.join(final_pattern) re_together = pattern_together for k, v in six.iteritems(TIME_CONVERSIONS): re_together = re_together.replace(k, v) # Create new empty lists final_date = list() final_pattern = list() for p, r in zip( pattern_together.split('%')[1:], re_together.split('\\')[1:]): regexp = '\\{}'.format(r) pattern = '%{}'.format(p) date_part = re.match(regexp, date_together) date_together = date_together[:date_part.start()] + \ date_together[date_part.end():] if pattern not in final_pattern: final_pattern.append('%{}'.format(p)) final_date.append(date_part.group()) return datetime.datetime.strptime(' '.join(final_date), ' '.join(final_pattern))
def _close(self, callback, args, server): """ Called after download is done. Activated queued downloads, call callback. """ callback(*args) self.connections[server] -= 1 self.conns -= 1 if self.q[server]: self._attempt_download(*self.q[server].pop()) else: for k, v in iteritems(self.q): # pylint: disable=W0612 while v: if self._attempt_download(*v[0]): v.popleft() if self.conns == self.max_total: return else: break
def _extractDateURL(self, url): """Extracts the date from a particular url following the pattern""" # url_to_list substitutes '.' and '_' for '/' to then create # a list of all the blocks in times - assuming they are all # separated with either '.', '_' or '/' url_to_list = lambda txt: re.sub(r'\.|_', '/', txt).split('/') pattern_list = url_to_list(self.pattern) url_list = url_to_list(url) time_order = ['%Y', '%y', '%b', '%B', '%m', '%d', '%j', '%H', '%I', '%M', '%S'] final_date = [] final_pattern = [] # Find in directory and filename for pattern_elem, url_elem in zip(pattern_list, url_list): time_formats = [x for x in time_order if x in pattern_elem] if len(time_formats) > 0: final_date.append(url_elem) final_pattern.append(pattern_elem) for time_bit in time_formats: time_order.remove(time_bit) # Find and remove repeated elements eg: %Y in ['%Y', '%Y%m%d'] # Make all as single strings date_together = ''.join(final_date) pattern_together = ''.join(final_pattern) re_together = pattern_together for k, v in six.iteritems(TIME_CONVERSIONS): re_together = re_together.replace(k, v) # Create new empty lists final_date = list() final_pattern = list() for p,r in zip(pattern_together.split('%')[1:], re_together.split('\\')[1:]): regexp = '\\{}'.format(r) pattern = '%{}'.format(p) date_part = re.match(regexp, date_together) date_together = date_together[:date_part.start()] + \ date_together[date_part.end():] if pattern not in final_pattern: final_pattern.append('%{}'.format(p)) final_date.append(date_part.group()) return datetime.datetime.strptime(' '.join(final_date), ' '.join(final_pattern))
def _close(self, callback, args, server): """ Called after download is done. Activated queued downloads, call callback. """ callback(*args) self.connections[server] -= 1 self.conns -= 1 if self.q[server]: self._attempt_download(*self.q[server].pop()) else: for k, v in iteritems(self.q): # pylint: disable=W0612 while v: if self._attempt_download(*v[0]): v.popleft() if self.conns == self.max_total: return else: break
def print_all(key=None): """ Provides a table of the complete list of constants. Parameters ---------- key : Python string or unicode Key in dictionary `constants` Returns ------- table : `astropy.table.Table` """ data_rows = [] for key, this_constant in iteritems(constants): data_rows.append([key, this_constant.name, this_constant.value, this_constant.uncertainty, str(this_constant.unit), this_constant.reference]) t = Table(rows=data_rows, names=('key', 'name', 'value', 'uncertainty', 'unit', 'Reference')) return t
def _regex_parse_time(inp, format): # Parser for finding out the minute value so we can adjust the string # from 24:00:00 to 00:00:00 the next day because strptime does not # understand the former. for key, value in six.iteritems(REGEX): format = format.replace(key, value) match = re.match(format, inp) if match is None: return None, None try: hour = match.group("hour") except IndexError: return inp, timedelta(days=0) if match.group("hour") == "24": if not all(_n_or_eq(_group_or_none(match, g, int), 00) for g in ["minute", "second", "microsecond"] ): raise ValueError from_, to = match.span("hour") return inp[:from_] + "00" + inp[to:], timedelta(days=1) return inp, timedelta(days=0)
def create_getdatarequest(self, maps, methods, info=None): """ Create datarequest from maps mapping data provider to fileids and methods, """ if info is None: info = {} # For the JSOC provider we need to make a DataRequestItem for each # series, not just one for the whole provider. # Remove JSOC provider items from the map jsoc = maps.pop('JSOC', []) # Make DRIs for everything that's not JSOC one per provider dris = [self.make('DataRequestItem', provider=k, fileiditem__fileid=[v]) for k, v in iteritems(maps)] def series_func(x): """ Extract the series from the fileid. """ return x.split(':')[0] # Sort the JSOC fileids by series # This is a precursor to groupby as recommended by the groupby docs series_sorted = sorted(jsoc, key=series_func) # Iterate over the series and make a DRI for each. # groupby creates an iterator based on a key function, in this case # based on the series (the part before the first ':') for series, fileids in itertools.groupby(series_sorted, key=series_func): dris.append(self.make('DataRequestItem', provider='JSOC', fileiditem__fileid=[list(fileids)])) return self.make( 'VSOGetDataRequest', request__method__methodtype=methods, request__info=info, request__datacontainer__datarequestitem=dris )
def entries_from_file(file, default_waveunit=None): """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> from sunpy.database.tables import entries_from_file >>> import sunpy.data >>> sunpy.data.download_sample_data(overwrite=False) # doctest: +SKIP >>> import sunpy.data.sample >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) >>> len(entries) 1 >>> entry = entries.pop() >>> entry.instrument 'SWAP' >>> entry.observation_time_start, entry.observation_time_end (datetime.datetime(2012, 1, 1, 0, 16, 7, 836000), None) >>> entry.wavemin, entry.wavemax (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) 111 """ headers = fits.get_header(file) if isinstance(file, (str, six.text_type)): filename = file else: filename = getattr(file, 'name', None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in six.iteritems(header): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == '': value = str(value) elif key == 'KEYCOMMENTS': for k, v in six.iteritems(value): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == 'INSTRUME': entry.instrument = value elif key == 'WAVELNTH': if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to( nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ('DATE-END', 'DATE_END'): entry.observation_time_end = parse_time(value) elif key in ('DATE-OBS', 'DATE_OBS'): entry.observation_time_start = parse_time(value) yield entry
from __future__ import absolute_import from sunpy.sun import constants as con from astropy.constants import Constant import pytest from sunpy.extern.six import iteritems @pytest.mark.parametrize('this_constant', [value for key, value in iteritems(con.constants)]) def test_all_constants_are_constants(this_constant): """Test that each member of the constants dict is an astropy Constant""" assert type(this_constant) is Constant @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_get_function(this_key): """Test that the get function works for all the keys""" assert type(con.get(this_key)) is Constant @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_find_function(this_key): """Test that the find function works for all the keys""" assert len(con.find(this_key)) >= 1 @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_find_function(this_key):
def __call__(self): for k, v in six.iteritems(self.kwargs): # save those values in the dict prev_values that will be changed # so that they can be recovered self.prev_values[k] = getattr(self.database_entry, k) setattr(self.database_entry, k, v)
# pylint: disable=E0102,C0103,W0613 def _create(wlk, root, api): """ Implementation detail. """ blocks = [] for attr in root.attrs: blocks.extend(wlk.create(attr, api)) return blocks # Converters take a type unknown to the walker and convert it into one # known to it. All of those convert types into ValueAttrs, which are # handled above by just assigning according to the keys and values of the # attrs member. walker.add_converter(Extent)( lambda x: ValueAttr( dict((('extent', k), v) for k, v in iteritems(vars(x))) ) ) walker.add_converter(Time)( lambda x: ValueAttr({ ('time', 'start'): x.start.strftime(TIMEFORMAT), ('time', 'end'): x.end.strftime(TIMEFORMAT) , ('time', 'near'): ( x.near.strftime(TIMEFORMAT) if x.near is not None else None), }) ) walker.add_converter(_VSOSimpleAttr)( lambda x: ValueAttr({(x.__class__.__name__.lower(), ): x.value}) )
def undo(self): for k, v in six.iteritems(self.prev_values): setattr(self.database_entry, k, v)
def __hash__(self): return hash(frozenset(iteritems(vars(self))))
def __hash__(self): return hash(frozenset(iteritems(self.attrs.iteritems)))
from __future__ import absolute_import from sunpy.sun import constants as con from astropy.constants import Constant import pytest from sunpy.extern.six import iteritems @pytest.mark.parametrize('this_constant', [value for key, value in iteritems(con.constants)]) def test_all_constants_are_constants(this_constant): """Test that each member of the constants dict is an astropy Constant""" assert isinstance(this_constant, Constant) @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_get_function(this_key): """Test that the get function works for all the keys""" assert isinstance(con.get(this_key), Constant) @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_find_function(this_key): """Test that the find function works for all the keys""" assert len(con.find(this_key)) >= 1 @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_find_function(this_key): """Test that the find function works for all the keys""" assert len(con.find(this_key)) >= 1
def entries_from_file(file, default_waveunit=None, time_string_parse_format=None): """Use the headers of a FITS file to generate an iterator of :class:`sunpy.database.tables.DatabaseEntry` instances. Gathered information will be saved in the attribute `fits_header_entries`. If the key INSTRUME, WAVELNTH or DATE-OBS / DATE_OBS is available, the attribute `instrument`, `wavemin` and `wavemax` or `observation_time_start` is set, respectively. If the wavelength unit can be read, the values of `wavemin` and `wavemax` are converted to nm (nanometres). The value of the `file` parameter is used to set the attribute `path` of each generated database entry. Parameters ---------- file : str or file-like object Either a path pointing to a FITS file or a an opened file-like object. If an opened file object, its mode must be one of the following rb, rb+, or ab+. default_waveunit : str, optional The wavelength unit that is used for a header if it cannot be found. time_string_parse_format : str, optional Fallback timestamp format which will be passed to `~datetime.datetime.strftime` if `sunpy.time.parse_time` is unable to automatically read the `date-obs` metadata. Raises ------ sunpy.database.WaveunitNotFoundError If `default_waveunit` is not given and the wavelength unit cannot be found in one of the FITS headers sunpy.WaveunitNotConvertibleError If a wavelength unit could be found but cannot be used to create an instance of the type ``astropy.units.Unit``. This can be the case for example if a FITS header has the key `WAVEUNIT` with the value `nonsense`. Examples -------- >>> from sunpy.database.tables import entries_from_file >>> import sunpy.data.sample # doctest: +REMOTE_DATA >>> entries = list(entries_from_file(sunpy.data.sample.SWAP_LEVEL1_IMAGE)) # doctest: +REMOTE_DATA >>> len(entries) # doctest: +REMOTE_DATA 1 >>> entry = entries.pop() # doctest: +REMOTE_DATA >>> entry.instrument # doctest: +REMOTE_DATA 'SWAP' >>> entry.observation_time_start, entry.observation_time_end # doctest: +REMOTE_DATA (datetime.datetime(2011, 6, 7, 6, 33, 29, 759000), None) >>> entry.wavemin, entry.wavemax # doctest: +REMOTE_DATA (17.400000000000002, 17.400000000000002) >>> len(entry.fits_header_entries) # doctest: +REMOTE_DATA 111 """ headers = fits.get_header(file) if isinstance(file, (str, six.text_type)): filename = file else: filename = getattr(file, 'name', None) for header in headers: entry = DatabaseEntry(path=filename) for key, value in six.iteritems(header): # Yes, it is possible to have an empty key in a FITS file. # Example: sunpy.data.sample.EIT_195_IMAGE # Don't ask me why this could be a good idea. if key == '': value = str(value) elif key == 'KEYCOMMENTS': for k, v in six.iteritems(value): entry.fits_key_comments.append(FitsKeyComment(k, v)) continue entry.fits_header_entries.append(FitsHeaderEntry(key, value)) waveunit = fits.extract_waveunit(header) entry.hdu_index = headers.index(header) if waveunit is None: waveunit = default_waveunit unit = None if waveunit is not None: try: unit = Unit(waveunit) except ValueError: raise WaveunitNotConvertibleError(waveunit) for header_entry in entry.fits_header_entries: key, value = header_entry.key, header_entry.value if key == 'INSTRUME': entry.instrument = value elif key == 'WAVELNTH': if unit is None: raise WaveunitNotFoundError(file) # use the value of `unit` to convert the wavelength to nm entry.wavemin = entry.wavemax = unit.to( nm, value, equivalencies.spectral()) # NOTE: the key DATE-END or DATE_END is not part of the official # FITS standard, but many FITS files use it in their header elif key in ('DATE-END', 'DATE_END'): entry.observation_time_end = parse_time( value, _time_string_parse_format=time_string_parse_format) elif key in ('DATE-OBS', 'DATE_OBS'): entry.observation_time_start = parse_time( value, _time_string_parse_format=time_string_parse_format) yield entry
def create_getdatarequest(self, maps, methods, info=None): """ Create datarequest from maps mapping data provider to fileids and methods, """ if info is None: info = {} if maps.get('JSOC',0): return self._separate_hmi(maps, methods, info) else: return self.make( 'VSOGetDataRequest', request__method__methodtype=methods, request__info=info, request__datacontainer__datarequestitem=[ self.make('DataRequestItem', provider=k, fileiditem__fileid=[v]) for k, v in iteritems(maps) ] ) def _separate_hmi(self, maps_dict, methods, info): """ This function parses the separate hmi data series from the maps_dict(jsoc) list, pulls any hmi data it finds into separate lists and deletes the hmi data from the original jsoc list. It then continues as normal to construct the soap message. """ jsoc_list=maps_dict.get('JSOC',0) if jsoc_list:
def query_legacy(self, tstart=None, tend=None, **kwargs): """ Query data from the VSO mocking the IDL API as close as possible. Either tstart and tend or date_start and date_end or date have to be supplied. Parameters ---------- tstart : datetime.datetime Start of the time-range in which records are searched. tend : datetime.datetime Start of the time-range in which records are searched. date : str (start date) - (end date) start_date : datetime the start date end_date : datetime the end date wave : str (min) - (max) (unit) min_wave : str minimum spectral range max_wave : str maximum spectral range unit_wave : str spectral range units (Angstrom, GHz, keV) extent : str VSO 'extent type' ... (FULLDISK, CORONA, LIMB, etc) physobj : str VSO 'physical observable' provider : str VSO ID for the data provider (SDAC, NSO, SHA, MSU, etc) source : str spacecraft or observatory (SOHO, YOHKOH, BBSO, etc) synonyms : spacecraft, observatory instrument : str instrument ID (EIT, SXI-0, SXT, etc) synonyms : telescope, inst detector : str detector ID (C3, EUVI, COR2, etc.) layout : str layout of the data (image, spectrum, time_series, etc.) level : str level of the data product (numeric range, see below) pixels : str number of pixels (numeric range, see below) resolution : str effective resolution (1 = full, 0.5 = 2x2 binned, etc) numeric range, see below. pscale : str pixel scale, in arcseconds (numeric range, see below) near_time : datetime return record closest to the time. See below. sample : int attempt to return only one record per SAMPLE seconds. See below. Numeric Ranges: - May be entered as a string or any numeric type for equality matching - May be a string of the format '(min) - (max)' for range matching - May be a string of the form '(operator) (number)' where operator is one of: lt gt le ge < > <= >= Examples -------- Query all data from eit between 2010-01-01T00:00 and 2010-01-01T01:00. >>> from datetime import datetime >>> from sunpy.net import vso >>> client = vso.VSOClient() # doctest: +REMOTE_DATA >>> qr = client.query_legacy(datetime(2010, 1, 1), ... datetime(2010, 1, 1, 1), instrument='eit') # doctest: +REMOTE_DATA Returns ------- out : :py:class:`QueryResult` (enhanced list) Matched items. Return value is of same type as the one of :py:class:`VSOClient.query`. """ sdk = lambda key: lambda value: {key: value} ALIASES = { 'wave_min': sdk('wave_wavemin'), 'wave_max': sdk('wave_wavemax'), 'wave_type': sdk('wave_wavetype'), 'wave_unit': sdk('wave_waveunit'), 'min_wave': sdk('wave_wavemin'), 'max_wave': sdk('wave_wavemax'), 'type_wave': sdk('wave_wavetype'), 'unit_wave': sdk('wave_waveunit'), 'wave': _parse_waverange, 'inst': sdk('instrument'), 'telescope': sdk('instrument'), 'spacecraft': sdk('source'), 'observatory': sdk('source'), 'start_date': sdk('time_start'), 'end_date': sdk('time_end'), 'start': sdk('time_start'), 'end': sdk('time_end'), 'near_time': sdk('time_near'), 'date': _parse_date, 'layout': sdk('datatype'), } if tstart is not None: kwargs.update({'time_start': tstart}) if tend is not None: kwargs.update({'time_end': tend}) queryreq = self.api.factory.create('QueryRequest') for key, value in iteritems(kwargs): for k, v in iteritems(ALIASES.get(key, sdk(key))(value)): if k.startswith('time'): v = parse_time(v).strftime(TIMEFORMAT) attr = k.split('_') lst = attr[-1] rest = attr[:-1] # pylint: disable=E1103 item = queryreq.block for elem in rest: try: item = item[elem] except KeyError: raise ValueError( "Unexpected argument {key!s}.".format(key=key)) if lst not in item: raise ValueError( "Unexpected argument {key!s}.".format(key=key)) if item[lst]: raise ValueError( "Got multiple values for {k!s}.".format(k=k)) item[lst] = v try: return QueryResponse.create(self.api.service.Query(queryreq)) except TypeNotFound: return QueryResponse([])
@walker.add_creator(AttrOr) # pylint: disable=E0102,C0103,W0613 def _create(wlk, root, api): """ Implementation detail. """ blocks = [] for attr in root.attrs: blocks.extend(wlk.create(attr, api)) return blocks # Converters take a type unknown to the walker and convert it into one # known to it. All of those convert types into ValueAttrs, which are # handled above by just assigning according to the keys and values of the # attrs member. walker.add_converter(Extent)(lambda x: ValueAttr( dict((('extent', k), v) for k, v in iteritems(vars(x))))) walker.add_converter(Time)(lambda x: ValueAttr({ ('time', 'start'): x.start.strftime(TIMEFORMAT), ('time', 'end'): x.end.strftime(TIMEFORMAT), ('time', 'near'): (x.near.strftime(TIMEFORMAT) if x.near is not None else None), })) walker.add_converter(_VSOSimpleAttr)( lambda x: ValueAttr({(x.__class__.__name__.lower(), ): x.value})) walker.add_converter(Wavelength)( lambda x: ValueAttr({
def __call__(self): for k, v in six.iteritems(self.kwargs): # save those values in the dict prev_values that will be changed # so that they can be recovered self.prev_values[k] = getattr(self.database_entry, k) setattr(self.database_entry, k, v)
def __hash__(self): return hash(frozenset(iteritems(self.attrs.iteritems)))
""" Create new SOAP object with attributes specified in kwargs. To assign subattributes, use foo__bar=1 to assign ['foo']['bar'] = 1. """ obj = self.api.factory.create(atype) for k, v in iteritems(kwargs): split = k.split('__') tip = split[-1] rest = split[:-1] item = obj for elem in rest: item = item[elem] if isinstance(v, dict): # Do not throw away type information for dicts. for k, v in iteritems(v): item[tip][k] = v else: item[tip] = v return obj def search(self, *query): """ Query data from the VSO with the new API. Takes a variable number of attributes as parameter, which are chained together using AND. The new query language allows complex queries to be easily formed. Examples -------- Query all data from eit or aia between 2010-01-01T00:00 and 2010-01-01T01:00.
def __hash__(self): return hash(frozenset(iteritems(vars(self))))
def undo(self): for k, v in six.iteritems(self.prev_values): setattr(self.database_entry, k, v)
from __future__ import absolute_import from sunpy.sun import constants as con from astropy.constants import Constant import pytest from sunpy.extern.six import iteritems @pytest.mark.parametrize('this_constant', [value for key, value in iteritems(con.constants)]) def test_all_constants_are_constants(this_constant): """Test that each member of the constants dict is an astropy Constant""" assert type(this_constant) is Constant @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_get_function(this_key): """Test that the get function works for all the keys""" assert type(con.get(this_key)) is Constant @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_find_function(this_key): """Test that the find function works for all the keys""" assert len(con.find(this_key)) >= 1 @pytest.mark.parametrize('this_key', [key for key, value in iteritems(con.constants)]) def test_find_function(this_key): """Test that the find function works for all the keys""" assert len(con.find(this_key)) >= 1
def iteritems(self): # pragma: no cover for key, value in six.iteritems(self._dict): yield key, value
def query_legacy(self, tstart=None, tend=None, **kwargs): """ Query data from the VSO mocking the IDL API as close as possible. Either tstart and tend or date_start and date_end or date have to be supplied. Parameters ---------- tstart : datetime.datetime Start of the time-range in which records are searched. tend : datetime.datetime Start of the time-range in which records are searched. date : str (start date) - (end date) start_date : datetime the start date end_date : datetime the end date wave : str (min) - (max) (unit) min_wave : str minimum spectral range max_wave : str maximum spectral range unit_wave : str spectral range units (Angstrom, GHz, keV) extent : str VSO 'extent type' ... (FULLDISK, CORONA, LIMB, etc) physobj : str VSO 'physical observable' provider : str VSO ID for the data provider (SDAC, NSO, SHA, MSU, etc) source : str spacecraft or observatory (SOHO, YOHKOH, BBSO, etc) synonyms : spacecraft, observatory instrument : str instrument ID (EIT, SXI-0, SXT, etc) synonyms : telescope, inst detector : str detector ID (C3, EUVI, COR2, etc.) layout : str layout of the data (image, spectrum, time_series, etc.) level : str level of the data product (numeric range, see below) pixels : str number of pixels (numeric range, see below) resolution : str effective resolution (1 = full, 0.5 = 2x2 binned, etc) numeric range, see below. pscale : str pixel scale, in arcseconds (numeric range, see below) near_time : datetime return record closest to the time. See below. sample : int attempt to return only one record per SAMPLE seconds. See below. Numeric Ranges: - May be entered as a string or any numeric type for equality matching - May be a string of the format '(min) - (max)' for range matching - May be a string of the form '(operator) (number)' where operator is one of: lt gt le ge < > <= >= Examples -------- Query all data from eit between 2010-01-01T00:00 and 2010-01-01T01:00. >>> from datetime import datetime >>> from sunpy.net import vso >>> client = vso.VSOClient() >>> qr = client.query_legacy(datetime(2010, 1, 1), ... datetime(2010, 1, 1, 1), instrument='eit') Returns ------- out : :py:class:`QueryResult` (enhanced list) of matched items. Return value of same type as the one of :py:class:`VSOClient.query`. """ sdk = lambda key: lambda value: {key: value} ALIASES = { 'wave_min': sdk('wave_wavemin'), 'wave_max': sdk('wave_wavemax'), 'wave_type': sdk('wave_wavetype'), 'wave_unit': sdk('wave_waveunit'), 'min_wave': sdk('wave_wavemin'), 'max_wave': sdk('wave_wavemax'), 'type_wave': sdk('wave_wavetype'), 'unit_wave': sdk('wave_waveunit'), 'wave': _parse_waverange, 'inst': sdk('instrument'), 'telescope': sdk('instrument'), 'spacecraft': sdk('source'), 'observatory': sdk('source'), 'start_date': sdk('time_start'), 'end_date': sdk('time_end'), 'start': sdk('time_start'), 'end': sdk('time_end'), 'near_time': sdk('time_near'), 'date': _parse_date, 'layout': sdk('datatype'), } if tstart is not None: kwargs.update({'time_start': tstart}) if tend is not None: kwargs.update({'time_end': tend}) queryreq = self.api.factory.create('QueryRequest') for key, value in iteritems(kwargs): for k, v in iteritems(ALIASES.get(key, sdk(key))(value)): if k.startswith('time'): v = parse_time(v).strftime(TIMEFORMAT) attr = k.split('_') lst = attr[-1] rest = attr[:-1] # pylint: disable=E1103 item = queryreq.block for elem in rest: try: item = item[elem] except KeyError: raise ValueError("Unexpected argument {key!s}.".format(key=key)) if lst not in item: raise ValueError("Unexpected argument {key!s}.".format(key=key)) if item[lst]: raise ValueError("Got multiple values for {k!s}.".format(k=k)) item[lst] = v try: return QueryResponse.create(self.api.service.Query(queryreq)) except TypeNotFound: return QueryResponse([])
def _create(wlk, root, session): query = session.query(DatabaseEntry) for key, value in six.iteritems(root.attrs): typ = key[0] if typ == 'tag': criterion = TableTag.name.in_([value]) # `key[1]` is here the `inverted` attribute of the tag. That means # that if it is True, the given tag must not be included in the # resulting entries. if key[1]: query = query.filter(~DatabaseEntry.tags.any(criterion)) else: query = query.filter(DatabaseEntry.tags.any(criterion)) elif typ == 'fitsheaderentry': key, val, inverted = value key_criterion = TableFitsHeaderEntry.key == key value_criterion = TableFitsHeaderEntry.value == val if inverted: query = query.filter( not_( and_( DatabaseEntry.fits_header_entries.any( key_criterion), DatabaseEntry.fits_header_entries.any( value_criterion)))) else: query = query.filter( and_( DatabaseEntry.fits_header_entries.any(key_criterion), DatabaseEntry.fits_header_entries.any( value_criterion))) elif typ == 'download time': start, end, inverted = value if inverted: query = query.filter( ~DatabaseEntry.download_time.between(start, end)) else: query = query.filter( DatabaseEntry.download_time.between(start, end)) elif typ == 'path': path, inverted = value if inverted: # pylint: disable=E711 query = query.filter( or_(DatabaseEntry.path != path, DatabaseEntry.path == None)) else: query = query.filter(DatabaseEntry.path == path) elif typ == 'wave': wavemin, wavemax, waveunit = value query = query.filter( and_(DatabaseEntry.wavemin >= wavemin, DatabaseEntry.wavemax <= wavemax)) elif typ == 'time': start, end, near = value query = query.filter( and_(DatabaseEntry.observation_time_start < end, DatabaseEntry.observation_time_end > start)) else: if typ.lower() not in SUPPORTED_SIMPLE_VSO_ATTRS.union( SUPPORTED_NONVSO_ATTRS): raise NotImplementedError( "The attribute {0!r} is not yet supported to query a database." .format(typ)) query = query.filter_by(**{typ: value}) return query.all()