def search(self, *args, **kwargs): extractor1 = '{}/warehouse/{:4d}/SRS/{year:4d}{month:2d}{day:2d}SRS.txt' extractor2 = '{}/warehouse/{year:4d}/{}' matchdict = self._get_match_dict(*args, **kwargs) timerange = matchdict['Time'] metalist = [] for url in self._get_url_for_timerange(timerange): exdict1 = parse(extractor1, url) exdict2 = parse(extractor2, url) exdict = (exdict2 if exdict1 is None else exdict1).named exdict['url'] = url rowdict = self.post_search_hook(exdict, matchdict) metalist.append(rowdict) return QueryResponse(metalist, client=self)
def _extract_files_meta(self, timerange, extractor, matcher=None): """ Returns metadata information contained in URLs. Parameters ---------- timerange : `~sunpy.time.TimeRange` Time interval where to find the directories for a given pattern. extractor: `str` Pattern to extract metadata by parsing the URL. matcher: `dict` Dictionary to check if extracted metadata is valid. Returns ------- `list` of `dict` List of metadata info for all URLs. """ self.extractor = extractor urls = self.filelist(timerange) metalist = [] for url in urls: metadict = parse(extractor, url) if metadict is not None: append = True metadict = metadict.named metadict['url'] = url if matcher is not None: for k in metadict: if k in matcher and str(metadict[k]) not in matcher[k]: append = False break if append: metalist.append(metadict) return metalist
def _check_timerange(self, url, timerange): """ Checks whether the time extracted from the URL is valid according to the given time range. Parameters ---------- url : `str` URL of the file. timerange : `~sunpy.time.TimeRange` Time interval for which files were searched. Returns ------- `bool` `True` if URL's time overlaps the given timerange, else `False`. """ if hasattr(self, 'extractor'): exdict = parse(self.extractor, url).named tr = get_timerange_from_exdict(exdict) return (tr.end >= timerange.start and tr.start <= timerange.end) else: datehref = self._extractDateURL(url).to_datetime() return (timerange.start.to_datetime() <= datehref <= timerange.end.to_datetime())
def _extract_files_meta(self, timerange, extractor=None): """ Returns metadata information contained in URLs. Parameters ---------- timerange : `~sunpy.time.TimeRange` Time interval where to find the directories for a given pattern. extractor: `str` Pattern to extract metadata by parsing the URL. Returns ------- metalist: `list` of `dict` List of metadata info for all URLs. """ urls = self.filelist(timerange) if extractor is None: return [{'url': url} for url in urls] metalist = [] for url in urls: metadict = parse(extractor, url) if metadict is not None: metadict = metadict.named else: metadict = {} metadict['url'] = url metalist.append(metadict) return metalist
def search(self, *args, **kwargs): baseurl, pattern, matchdict = self.pre_search_hook(*args, **kwargs) timerange = matchdict['Time'] metalist = [] for url in self.get_observing_summary_filename(timerange): exdict = parse(pattern, url).named exdict['url'] = url rowdict = super().post_search_hook(exdict, matchdict) metalist.append(rowdict) return QueryResponse(metalist, client=self)