def __call__(self):
self.logger.debug('Downloading (url={}, stream_epochs={}) ...'.format(
self._request_handler.url, self._request_handler.stream_epochs))
try:
with open(self.path_tempfile, 'wb') as ofd:
# NOTE(damb): For granular fdnsws-station-text request it seems
# ok buffering the entire response in memory.
with binary_request(self._request_handler.post()) as ifd:
for line in ifd:
self._size += len(line)
if line.startswith(b'#'):
continue
ofd.write(line.strip() + b'\n')
except RequestsError as err:
return self._handle_error(err)
else:
self.logger.debug(
'Download (url={}, stream_epochs={}) finished.'.format(
self._request_handler.url,
self._request_handler.stream_epochs))
return Result.ok(data=self.path_tempfile, length=self._size)
def config(self):
# proxy for fetching the config from the EIDA node
if self._config is None:
req = functools.partial(requests.get, self.url)
with binary_request(req) as resp:
self._config = resp
return self._config
def _route(self):
"""
Create the routing table using the routing service provided.
"""
routing_request = RoutingRequestHandler(self._routing_service,
self.query_params,
self.stream_epochs)
req = (routing_request.post() if self.post else routing_request.get())
self.logger.info("Fetching routes from %s" % routing_request.url)
routing_table = []
try:
with binary_request(req) as fd:
# parse the routing service's output stream; create a routing
# table
urlline = None
stream_epochs = []
while True:
line = fd.readline()
if not urlline:
urlline = line.strip()
elif not line.strip():
# set up the routing table
if stream_epochs:
routing_table.append(
utils.Route(url=urlline,
streams=stream_epochs))
urlline = None
stream_epochs = []
if not line:
break
else:
stream_epochs.append(
StreamEpoch.from_snclline(
line, default_endtime=self.DEFAULT_ENDTIME))
except NoContent as err:
self.logger.warning(err)
raise FDSNHTTPError.create(
int(
self.query_params.get(
'nodata',
settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE)))
except RequestsError as err:
self.logger.error(err)
raise FDSNHTTPError.create(500, service_version=__version__)
else:
self.logger.debug('Number of routes received: {}'.format(
len(routing_table)))
return routing_table
def _run(self, req):
"""
Extracts ``Network`` elements from StationXML.
"""
with open(self.path_tempfile, 'wb') as ofd:
# XXX(damb): Load the entire result into memory.
with binary_request(req, logger=self.logger) as ifd:
for event, net_element in etree.iterparse(
ifd, tag=self._network_tags):
if event == 'end':
s = etree.tostring(net_element)
self._size += len(s)
ofd.write(s)
def _run(self, req):
"""
Removes ``fdsnws-station`` ``format=text`` headers while downloading.
"""
with open(self.path_tempfile, 'wb') as ofd:
# NOTE(damb): For granular fdnsws-station-text request it seems
# ok buffering the entire response in memory.
with binary_request(req, logger=self.logger) as ifd:
for line in ifd:
self._size += len(line)
if line.startswith(b'#'):
continue
ofd.write(line.strip() + b'\n')
def _route(self, req, post=True, **kwargs):
"""
Route a request and create a routing table. Routing is performed by
means of the routing service provided.
:param req: Routing service request handler
:type req: :py:class:`RoutingRequestHandler`
:param bool post: Execute a the request to the routing service via HTTP
POST
:raises NoContent: If no routes are available
:raises RequestsError: General exception if request to routing service
failed
"""
_req = (req.post() if post else req.get())
routing_table = {}
self.logger.info("Fetching routes from %s" % req.url)
try:
with binary_request(_req) as fd:
# parse the routing service's output stream; create a routing
# table
urlline = None
stream_epochs = []
while True:
line = fd.readline()
if not urlline:
urlline = line.strip()
elif not line.strip():
# set up the routing table
if stream_epochs:
routing_table[urlline] = stream_epochs
urlline = None
stream_epochs = []
if not line:
break
else:
# XXX(damb): Do not substitute an empty endtime when
# performing HTTP GET requests in order to guarantee
# more cache hits (if eida-federator is coupled with
# HTTP caching proxy).
stream_epochs.append(
StreamEpoch.from_snclline(
line,
default_endtime=(self._default_endtime
if post else None)))
except NoContent as err:
self.logger.warning(err)
nodata = int(
kwargs.get('nodata',
settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE))
raise FDSNHTTPError.create(nodata)
except RequestsError as err:
self.logger.error(err)
raise FDSNHTTPError.create(500, service_version=__version__)
else:
self.logger.debug('Number of routes received: {}'.format(
len(routing_table)))
return routing_table
def harvest(self, session):
"""Harvest the routing configuration."""
route_tag = '{}route'.format(self.NS_ROUTINGXML)
_cached_services = get_cached_services()
_cached_services = [
'{}{}'.format(self.NS_ROUTINGXML, s) for s in _cached_services
]
self.logger.debug('Harvesting routes for %s.' % self.node)
# event driven parsing
for event, route_element in etree.iterparse(self.config,
events=('end', ),
tag=route_tag):
if event == 'end' and len(route_element):
stream = Stream.from_route_attrs(**dict(route_element.attrib))
attrs = dict(stream._asdict())
# create query parameters from stream attrs
query_params = '&'.join([
'{}={}'.format(query_param, query_val)
for query_param, query_val in attrs.items()
])
# extract fdsn-station service url for each route
urls = set([
e.get('address') for e in route_element.iter('{}{}'.format(
self.NS_ROUTINGXML, self.STATION_TAG))
if int(e.get('priority', 0)) == 1
])
if (len(urls) == 0 and len([
e for e in route_element.iter()
if int(e.get('priority', 0)) == 1
]) == 0):
# NOTE(damb): Skip routes which contain exclusively
# 'priority == 2' services
continue
elif len(urls) > 1:
# NOTE(damb): Currently we cannot handle multiple
# fdsn-station urls i.e. for multiple routed epochs
raise self.IntegrityError(
('Missing <station></station> element for '
'{} ({}).'.format(route_element, urls)))
_url_fdsn_station = '{}?{}&level=channel'.format(
urls.pop(), query_params)
# XXX(damb): For every single route resolve FDSN wildcards
# using the route's station service.
# XXX(damb): Use the station service's GET method since the
# POST method requires temporal constraints (both starttime and
# endtime).
# ----
self.logger.debug('Resolving routing: (Request: %r).' %
_url_fdsn_station)
nets = []
stas = []
chas = []
try:
# TODO(damb): Request might be too large. Implement fix.
req = functools.partial(requests.get, _url_fdsn_station)
with binary_request(req) as station_xml:
nets, stas, chas = \
self._harvest_from_stationxml(session, station_xml)
except NoContent as err:
self.logger.warning(str(err))
continue
# NOTE(damb): currently only consider CACHED_SERVICEs
for service_element in route_element.iter(*_cached_services):
# only consider priority=1
priority = service_element.get('priority')
if not priority or int(priority) != 1:
self.logger.info(
"Skipping {} due to priority '{}'.".format(
service_element, priority))
continue
# remove xml namespace
service_tag = service_element.tag[len(self.NS_ROUTINGXML):]
endpoint_url = service_element.get('address')
if not endpoint_url:
raise self.RoutingConfigXMLParsingError(
"Missing 'address' attrib.")
service = self._emerge_service(session, service_tag)
endpoint = self._emerge_endpoint(session, endpoint_url,
service)
self.logger.debug(
'Processing routes for %r '
'(service=%s, endpoint=%s).' %
(stream, service_element.tag, endpoint.url))
try:
routing_starttime = UTCDateTime(
service_element.get('start'),
iso8601=True).datetime
routing_endtime = service_element.get('end')
# reset endtime due to 'end=""'
routing_endtime = (
UTCDateTime(routing_endtime, iso8601=True).datetime
if routing_endtime is not None
and routing_endtime.strip() else None)
except Exception as err:
raise self.RoutingConfigXMLParsingError(err)
# configure routings
for cha_epoch in chas:
if inspect(cha_epoch).deleted:
# In case a orm.ChannelEpoch object is marked as
# deleted but harvested within the same harvesting
# run this is a strong hint for an integrity issue
# within the FDSN station InventoryXML.
msg = ('InventoryXML integrity issue for '
'{0!r}'.format(cha_epoch))
warnings.warn(msg)
self.logger.warning(msg)
continue
self.logger.debug(
'Checking ChannelEpoch<->Endpoint relation '
'{}<->{} ...'.format(cha_epoch, endpoint))
_ = self._emerge_routing(session, cha_epoch, endpoint,
routing_starttime,
routing_endtime)