def _route(self):
"""
Create the routing table using the routing service provided.
"""
routing_request = RoutingRequestHandler(self._routing_service,
self.query_params,
self.stream_epochs)
req = (routing_request.post() if self.post else routing_request.get())
self.logger.info("Fetching routes from %s" % routing_request.url)
routing_table = []
try:
with binary_request(req) as fd:
# parse the routing service's output stream; create a routing
# table
urlline = None
stream_epochs = []
while True:
line = fd.readline()
if not urlline:
urlline = line.strip()
elif not line.strip():
# set up the routing table
if stream_epochs:
routing_table.append(
utils.Route(url=urlline,
streams=stream_epochs))
urlline = None
stream_epochs = []
if not line:
break
else:
stream_epochs.append(
StreamEpoch.from_snclline(
line, default_endtime=self.DEFAULT_ENDTIME))
except NoContent as err:
self.logger.warning(err)
raise FDSNHTTPError.create(
int(
self.query_params.get(
'nodata',
settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE)))
except RequestsError as err:
self.logger.error(err)
raise FDSNHTTPError.create(500, service_version=__version__)
else:
self.logger.debug('Number of routes received: {}'.format(
len(routing_table)))
return routing_table
def handle_parser_error(err, req):
"""
configure keywordparser error handler
"""
raise FDSNHTTPError.create(400,
service_version=service_version,
error_desc_long=str(err))
def get(self, args, stream_epochs):
"""
Process a *WFCatalog* GET request.
"""
# request.method == 'GET'
# sanity check - starttime and endtime must be specified
if (not stream_epochs or stream_epochs[0].starttime is None or
stream_epochs[0].endtime is None):
raise FDSNHTTPError.create(
400, service_version=__version__,
error_desc_long='Both starttime and endtime required.')
self.logger.debug('StreamEpoch objects: %r' % stream_epochs)
# serialize objects
s = WFCatalogSchema()
args = s.dump(args)
self.logger.debug('WFCatalogSchema (serialized): %s' % args)
# process request
processor = RequestProcessor.create(
args['service'],
settings.WFCATALOG_MIMETYPE,
query_params=args,
stream_epochs=stream_epochs,
context=g.ctx,
keep_tempfiles=current_app.config['FED_KEEP_TEMPFILES'],
retry_budget_client=current_app.config['FED_CRETRY_BUDGET_ERATIO'],
**current_app.config['FED_RESOURCE_CONFIG']['eidaws-wfcatalog'],)
processor.post = False
return processor.streamed_response
def handle_parser_error(err, req, schema, error_status_code,
error_headers):
"""
configure webargs error handler
"""
raise FDSNHTTPError.create(400,
service_version=service_version,
error_desc_long=str(err))
def decorator(self, *args, **kwargs):
try:
return func(self, *args, **kwargs)
except FDSNHTTPError as err:
raise err
except Exception as err:
# NOTE(damb): Prevents displaying the full stack trace. Just log
# it.
exc_type, exc_value, exc_traceback = sys.exc_info()
self.logger.critical('Local Exception: %s' % type(err))
self.logger.critical('Traceback information: ' + repr(
traceback.format_exception(exc_type, exc_value, exc_traceback))
)
raise FDSNHTTPError.create(500, service_version=service_version)
def _wait(self, timeout=None):
"""
Wait for a valid endpoint response.
:param int timeout: Timeout in seconds
"""
if timeout is None:
timeout = self.TIMEOUT_STREAMING
result_with_data = False
while True:
ready = []
for result in self._results:
if result.ready():
_result = result.get()
if _result.status_code == 200:
result_with_data = True
break
elif _result.status_code == 413:
self._handle_413(_result)
ready.append(result)
else:
self._handle_error(_result)
self._sizes.append(0)
ready.append(result)
# NOTE(damb): We have to handle responses > 5MB. Blocking the
# processor by means of time.sleep makes executing
# *DownloadTasks IO bound.
time.sleep(0.01)
for result in ready:
self._results.remove(result)
if result_with_data:
break
if (not self._results
or datetime.datetime.utcnow() > self.DEFAULT_ENDTIME +
datetime.timedelta(seconds=timeout)):
self.logger.warning(
'No valid results to be federated. ({})'.format(
('No valid results.' if not self._results else
'Timeout ({}).'.format(timeout))))
raise FDSNHTTPError.create(
int(
self.query_params.get(
'nodata',
settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE)))
def _create_response(self, stream_wrapper=None, **wrapper_kwargs):
self._route()
if not self._num_routes:
raise FDSNHTTPError.create(self._nodata)
self._request()
# XXX(damb): Only return a streamed response as soon as valid data
# is available. Use a timeout and process errors here.
self._wait()
response_generator = stream_with_context(self)
if callable(stream_wrapper):
response_generator = stream_wrapper(
response_generator, **wrapper_kwargs)
resp = Response(response_generator, mimetype=self.mimetype,
content_type=self.content_type)
resp.call_on_close(self._call_on_close)
return resp
def get(self, args, stream_epochs):
"""
Process a *WFCatalog* GET request.
"""
# request.method == 'GET'
# sanity check - starttime and endtime must be specified
if (not stream_epochs or stream_epochs[0].starttime is None
or stream_epochs[0].endtime is None):
raise FDSNHTTPError.create(400, service_version=__version__)
self.logger.debug('StreamEpoch objects: %r' % stream_epochs)
# serialize objects
s = WFCatalogSchema()
args = s.dump(args)
self.logger.debug('WFCatalogSchema (serialized): %s' % args)
# process request
return RequestProcessor.create(args['service'],
settings.WFCATALOG_MIMETYPE,
query_params=args,
stream_epochs=stream_epochs,
post=False).streamed_response
def _route(self, req, post=True, **kwargs):
"""
Route a request and create a routing table. Routing is performed by
means of the routing service provided.
:param req: Routing service request handler
:type req: :py:class:`RoutingRequestHandler`
:param bool post: Execute a the request to the routing service via HTTP
POST
:raises NoContent: If no routes are available
:raises RequestsError: General exception if request to routing service
failed
"""
_req = (req.post() if post else req.get())
routing_table = {}
self.logger.info("Fetching routes from %s" % req.url)
try:
with binary_request(_req) as fd:
# parse the routing service's output stream; create a routing
# table
urlline = None
stream_epochs = []
while True:
line = fd.readline()
if not urlline:
urlline = line.strip()
elif not line.strip():
# set up the routing table
if stream_epochs:
routing_table[urlline] = stream_epochs
urlline = None
stream_epochs = []
if not line:
break
else:
# XXX(damb): Do not substitute an empty endtime when
# performing HTTP GET requests in order to guarantee
# more cache hits (if eida-federator is coupled with
# HTTP caching proxy).
stream_epochs.append(
StreamEpoch.from_snclline(
line,
default_endtime=(self._default_endtime
if post else None)))
except NoContent as err:
self.logger.warning(err)
nodata = int(
kwargs.get('nodata',
settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE))
raise FDSNHTTPError.create(nodata)
except RequestsError as err:
self.logger.error(err)
raise FDSNHTTPError.create(500, service_version=__version__)
else:
self.logger.debug('Number of routes received: {}'.format(
len(routing_table)))
return routing_table
def _handle_413(self, result):
self.logger.warning('Handle endpoint HTTP status code 413 (url={}, '
'stream_epochs={}).'.format(
result.data.url, result.data.stream_epochs))
raise FDSNHTTPError.create(413, service_version=__version__)
def _handle_nodata(self, args):
raise FDSNHTTPError.create(
int(
args.get('nodata',
settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE)))
def handle_parser_error(err, req):
"""
configure webargs error handler
"""
raise FDSNHTTPError.create(400, service_version=service_version)