def __call__(self): self.logger.debug('Downloading (url={}, stream_epochs={}) ...'.format( self._request_handler.url, self._request_handler.stream_epochs)) try: with open(self.path_tempfile, 'wb') as ofd: # NOTE(damb): For granular fdnsws-station-text request it seems # ok buffering the entire response in memory. with binary_request(self._request_handler.post()) as ifd: for line in ifd: self._size += len(line) if line.startswith(b'#'): continue ofd.write(line.strip() + b'\n') except RequestsError as err: return self._handle_error(err) else: self.logger.debug( 'Download (url={}, stream_epochs={}) finished.'.format( self._request_handler.url, self._request_handler.stream_epochs)) return Result.ok(data=self.path_tempfile, length=self._size)
def config(self): # proxy for fetching the config from the EIDA node if self._config is None: req = functools.partial(requests.get, self.url) with binary_request(req) as resp: self._config = resp return self._config
def _route(self): """ Create the routing table using the routing service provided. """ routing_request = RoutingRequestHandler(self._routing_service, self.query_params, self.stream_epochs) req = (routing_request.post() if self.post else routing_request.get()) self.logger.info("Fetching routes from %s" % routing_request.url) routing_table = [] try: with binary_request(req) as fd: # parse the routing service's output stream; create a routing # table urlline = None stream_epochs = [] while True: line = fd.readline() if not urlline: urlline = line.strip() elif not line.strip(): # set up the routing table if stream_epochs: routing_table.append( utils.Route(url=urlline, streams=stream_epochs)) urlline = None stream_epochs = [] if not line: break else: stream_epochs.append( StreamEpoch.from_snclline( line, default_endtime=self.DEFAULT_ENDTIME)) except NoContent as err: self.logger.warning(err) raise FDSNHTTPError.create( int( self.query_params.get( 'nodata', settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE))) except RequestsError as err: self.logger.error(err) raise FDSNHTTPError.create(500, service_version=__version__) else: self.logger.debug('Number of routes received: {}'.format( len(routing_table))) return routing_table
def _run(self, req): """ Extracts ``Network`` elements from StationXML. """ with open(self.path_tempfile, 'wb') as ofd: # XXX(damb): Load the entire result into memory. with binary_request(req, logger=self.logger) as ifd: for event, net_element in etree.iterparse( ifd, tag=self._network_tags): if event == 'end': s = etree.tostring(net_element) self._size += len(s) ofd.write(s)
def _run(self, req): """ Removes ``fdsnws-station`` ``format=text`` headers while downloading. """ with open(self.path_tempfile, 'wb') as ofd: # NOTE(damb): For granular fdnsws-station-text request it seems # ok buffering the entire response in memory. with binary_request(req, logger=self.logger) as ifd: for line in ifd: self._size += len(line) if line.startswith(b'#'): continue ofd.write(line.strip() + b'\n')
def _route(self, req, post=True, **kwargs): """ Route a request and create a routing table. Routing is performed by means of the routing service provided. :param req: Routing service request handler :type req: :py:class:`RoutingRequestHandler` :param bool post: Execute a the request to the routing service via HTTP POST :raises NoContent: If no routes are available :raises RequestsError: General exception if request to routing service failed """ _req = (req.post() if post else req.get()) routing_table = {} self.logger.info("Fetching routes from %s" % req.url) try: with binary_request(_req) as fd: # parse the routing service's output stream; create a routing # table urlline = None stream_epochs = [] while True: line = fd.readline() if not urlline: urlline = line.strip() elif not line.strip(): # set up the routing table if stream_epochs: routing_table[urlline] = stream_epochs urlline = None stream_epochs = [] if not line: break else: # XXX(damb): Do not substitute an empty endtime when # performing HTTP GET requests in order to guarantee # more cache hits (if eida-federator is coupled with # HTTP caching proxy). stream_epochs.append( StreamEpoch.from_snclline( line, default_endtime=(self._default_endtime if post else None))) except NoContent as err: self.logger.warning(err) nodata = int( kwargs.get('nodata', settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE)) raise FDSNHTTPError.create(nodata) except RequestsError as err: self.logger.error(err) raise FDSNHTTPError.create(500, service_version=__version__) else: self.logger.debug('Number of routes received: {}'.format( len(routing_table))) return routing_table
def harvest(self, session): """Harvest the routing configuration.""" route_tag = '{}route'.format(self.NS_ROUTINGXML) _cached_services = get_cached_services() _cached_services = [ '{}{}'.format(self.NS_ROUTINGXML, s) for s in _cached_services ] self.logger.debug('Harvesting routes for %s.' % self.node) # event driven parsing for event, route_element in etree.iterparse(self.config, events=('end', ), tag=route_tag): if event == 'end' and len(route_element): stream = Stream.from_route_attrs(**dict(route_element.attrib)) attrs = dict(stream._asdict()) # create query parameters from stream attrs query_params = '&'.join([ '{}={}'.format(query_param, query_val) for query_param, query_val in attrs.items() ]) # extract fdsn-station service url for each route urls = set([ e.get('address') for e in route_element.iter('{}{}'.format( self.NS_ROUTINGXML, self.STATION_TAG)) if int(e.get('priority', 0)) == 1 ]) if (len(urls) == 0 and len([ e for e in route_element.iter() if int(e.get('priority', 0)) == 1 ]) == 0): # NOTE(damb): Skip routes which contain exclusively # 'priority == 2' services continue elif len(urls) > 1: # NOTE(damb): Currently we cannot handle multiple # fdsn-station urls i.e. for multiple routed epochs raise self.IntegrityError( ('Missing <station></station> element for ' '{} ({}).'.format(route_element, urls))) _url_fdsn_station = '{}?{}&level=channel'.format( urls.pop(), query_params) # XXX(damb): For every single route resolve FDSN wildcards # using the route's station service. # XXX(damb): Use the station service's GET method since the # POST method requires temporal constraints (both starttime and # endtime). # ---- self.logger.debug('Resolving routing: (Request: %r).' % _url_fdsn_station) nets = [] stas = [] chas = [] try: # TODO(damb): Request might be too large. Implement fix. req = functools.partial(requests.get, _url_fdsn_station) with binary_request(req) as station_xml: nets, stas, chas = \ self._harvest_from_stationxml(session, station_xml) except NoContent as err: self.logger.warning(str(err)) continue # NOTE(damb): currently only consider CACHED_SERVICEs for service_element in route_element.iter(*_cached_services): # only consider priority=1 priority = service_element.get('priority') if not priority or int(priority) != 1: self.logger.info( "Skipping {} due to priority '{}'.".format( service_element, priority)) continue # remove xml namespace service_tag = service_element.tag[len(self.NS_ROUTINGXML):] endpoint_url = service_element.get('address') if not endpoint_url: raise self.RoutingConfigXMLParsingError( "Missing 'address' attrib.") service = self._emerge_service(session, service_tag) endpoint = self._emerge_endpoint(session, endpoint_url, service) self.logger.debug( 'Processing routes for %r ' '(service=%s, endpoint=%s).' % (stream, service_element.tag, endpoint.url)) try: routing_starttime = UTCDateTime( service_element.get('start'), iso8601=True).datetime routing_endtime = service_element.get('end') # reset endtime due to 'end=""' routing_endtime = ( UTCDateTime(routing_endtime, iso8601=True).datetime if routing_endtime is not None and routing_endtime.strip() else None) except Exception as err: raise self.RoutingConfigXMLParsingError(err) # configure routings for cha_epoch in chas: if inspect(cha_epoch).deleted: # In case a orm.ChannelEpoch object is marked as # deleted but harvested within the same harvesting # run this is a strong hint for an integrity issue # within the FDSN station InventoryXML. msg = ('InventoryXML integrity issue for ' '{0!r}'.format(cha_epoch)) warnings.warn(msg) self.logger.warning(msg) continue self.logger.debug( 'Checking ChannelEpoch<->Endpoint relation ' '{}<->{} ...'.format(cha_epoch, endpoint)) _ = self._emerge_routing(session, cha_epoch, endpoint, routing_starttime, routing_endtime)