def request(self, pool, tasks, query_params={}, **kwargs):
"""
Issue a bulk endpoint request with network code granularity. For
distributed physical networks a request execution is delegated to a
secondary-level task.
"""
assert hasattr(self, '_routes'), 'Missing routes.'
default_task = self._get_task_by_kw(tasks, 'default')
combining_task = self._get_task_by_kw(tasks, 'combining')
http_method = kwargs.pop('http_method',
settings.EIDA_FEDERATOR_DEFAULT_HTTP_METHOD)
retval = []
for net, routes in self._routes.items():
# create subcontext
ctx = Context()
self._ctx.append(ctx)
if len(routes) == 1:
if http_method == 'GET':
self.logger.debug('Force HTTP POST endpoint requests.')
self.logger.debug('Creating {!r} for net={!r} ...'.format(
default_task, net))
# NOTE(damb): For bulk requests there's only http_method='POST'
t = default_task(BulkFdsnRequestHandler(
routes[0].url,
stream_epochs=routes[0].streams,
query_params=query_params),
context=ctx,
name=net,
http_method='POST',
**kwargs)
elif len(routes) > 1:
self.logger.debug('Creating {!r} for net={!r} ...'.format(
combining_task, net))
t = combining_task(routes,
query_params,
name=net,
context=ctx,
http_method=http_method,
**kwargs)
else:
raise RoutingError('Missing routes.')
result = pool.apply_async(t)
retval.append(result)
return retval
def _handle_413(self, result):
self.logger.info(
'Handle endpoint HTTP status code 413 (url={}, '
'stream_epochs={}).'.format(result.data.url,
result.data.stream_epochs))
self.logger.debug(
'Creating SAATask for (url={}, '
'stream_epochs={}) ...'.format(result.data.url,
result.data.stream_epochs))
ctx = Context()
self._ctx.append(ctx)
t = WFCatalogSplitAndAlignTask(
result.data.url, result.data.stream_epochs[0],
query_params=self.query_params,
endtime=self.DEFAULT_ENDTIME,
context=ctx,
keep_tempfiles=self._keep_tempfiles,)
result = self._pool.apply_async(t)
self._results.append(result)
def request(self, pool, tasks, query_params={}, **kwargs):
"""
Issue granular endpoint requests.
"""
assert hasattr(self, '_routes'), 'Missing routes.'
default_task = self._get_task_by_kw(tasks, 'default')
retval = []
for route in self._routes:
self.logger.debug('Creating {!r} for {!r} ...'.format(
default_task, route))
ctx = Context()
self._ctx.append(ctx)
t = default_task(GranularFdsnRequestHandler(
route.url, route.streams[0], query_params=query_params),
context=ctx,
**kwargs)
result = pool.apply_async(t)
retval.append(result)
return retval
def request(self, pool, tasks, query_params={}, **kwargs):
"""
Issue a bulk endpoint request with network granularity.
"""
assert hasattr(self, '_routes'), 'Missing routes.'
default_task = self._get_task_by_kw(tasks, 'default')
http_method = kwargs.pop('http_method',
settings.EIDA_FEDERATOR_DEFAULT_HTTP_METHOD)
if http_method == 'GET':
self.logger.debug('Force HTTP POST endpoint requests.')
retval = []
for net, bulk_routes in self._routes.items():
self.logger.debug('Creating tasks for net={!r} ...'.format(net))
for bulk_route in bulk_routes:
self.logger.debug('Creating {!r} for {!r} ...'.format(
default_task, bulk_route))
ctx = Context()
self._ctx.append(ctx)
# NOTE(damb): For bulk requests there's only http_method='POST'
t = default_task(BulkFdsnRequestHandler(
bulk_route.url,
stream_epochs=bulk_route.streams,
query_params=query_params),
context=ctx,
http_method='POST',
**kwargs)
result = pool.apply_async(t)
retval.append(result)
return retval
def request(self, pool, tasks, query_params={}, **kwargs):
"""
Issue combining tasks. Issuing endpoint requests is delegated to those
tasks.
"""
assert hasattr(self, '_routes'), 'Missing routes.'
combining_task = self._get_task_by_kw(tasks, 'combining')
retval = []
for net, routes in self._routes.items():
ctx = Context()
self._ctx.append(ctx)
self.logger.debug('Creating {!r} for net={!r} ...'.format(
combining_task, net))
t = combining_task(routes,
query_params,
name=net,
context=ctx,
**kwargs)
result = pool.apply_async(t)
retval.append(result)
return retval
def before_request():
g.request_start_time = datetime.datetime.utcnow()
g.request_id = uuid.uuid4()
g.ctx = Context(ctx=g.request_id)
g.ctx.acquire()
def _run(self):
"""
Combine `StationXML <http://www.fdsn.org/xml/station/>`_
:code:`<Network></Network>` information.
"""
self.logger.info('Executing task {!r} ...'.format(self))
self._pool = ThreadPool(processes=self._num_workers)
for route in self._routes:
self.logger.debug(
'Creating DownloadTask for route {!r} ...'.format(route))
ctx = Context()
self._ctx.append(ctx)
t = RawDownloadTask(GranularFdsnRequestHandler(
route.url, route.streams[0], query_params=self.query_params),
decode_unicode=True,
context=ctx,
keep_tempfiles=self._keep_tempfiles,
http_method=self._http_method)
# apply DownloadTask asynchronoulsy to the worker pool
result = self._pool.apply_async(t)
self._results.append(result)
self._pool.close()
# fetch results ready
while True:
ready = []
for result in self._results:
if result.ready():
_result = result.get()
if _result.status_code == 200:
if self._level in ('channel', 'response'):
# merge <Channel></Channel> elements into
# <Station></Station> from the correct
# <Network></Network> epoch element
for _net_element in self._extract_net_elements(
_result.data):
# find the correct <Network></Network> epoch
# element
net_element, known = self._emerge_net_element(
_net_element,
exclude_tags=[
'{}{}'.format(ns, self.STATION_TAG) for
ns in settings.STATIONXML_NAMESPACES
])
if not known:
continue
# append/merge station elements
for sta_element in \
self._emerge_sta_elements(
_net_element):
self._merge_sta_element(
net_element, sta_element)
elif self._level == 'station':
# append <Station></Station> elements to the
# corresponding <Network></Network> epoch
for _net_element in self._extract_net_elements(
_result.data):
net_element, known = self._emerge_net_element(
_net_element,
exclude_tags=[
'{}{}'.format(ns, self.STATION_TAG) for
ns in settings.STATIONXML_NAMESPACES
])
if not known:
continue
# append station elements
# NOTE(damb): <Station></Station> elements
# defined by multiple EIDA nodes are simply
# appended; no merging is performed
for sta_element in \
self._emerge_sta_elements(
_net_element):
net_element.append(sta_element)
elif self._level == 'network':
for net_element in self._extract_net_elements(
_result.data):
_, _ = self._emerge_net_element(net_element)
self._clean(_result)
self._sizes.append(_result.length)
else:
self._handle_error(_result)
self._sizes.append(0)
ready.append(result)
for result in ready:
self._results.remove(result)
if not self._results:
break
if self._has_inactive_ctx():
self.logger.debug('{}: Closing ...'.format(self.name))
self._terminate()
raise self.MissingContextLock
self._pool.join()
if not sum(self._sizes):
self.logger.warning(
'Task {!r} terminates with no valid result.'.format(self))
return Result.nocontent(extras={'type_task': self._TYPE})
_length = 0
# dump xml tree for <Network></Network> epochs to temporary file
self.path_tempfile = get_temp_filepath()
self.logger.debug('{}: tempfile={!r}'.format(self, self.path_tempfile))
with open(self.path_tempfile, 'wb') as ofd:
for net_element in self._network_elements:
s = etree.tostring(net_element)
_length += len(s)
ofd.write(s)
if self._has_inactive_ctx():
raise self.MissingContextLock
self.logger.info(
('Task {!r} sucessfully finished '
'(total bytes processed: {}, after processing: {}).').format(
self, sum(self._sizes), _length))
return Result.ok(data=self.path_tempfile,
length=_length,
extras={'type_task': self._TYPE})
def __init__(self, mimetype, query_params={}, stream_epochs=[], **kwargs):
"""
:param str mimetype: The response's mimetype
:param dict query_params: Request query parameters
:param stream_epochs: Stream epochs requested
:type stream_epochs: List of :py:class:`StreamEpoch`
:param str logger: Logger name (optional)
:param request_strategy: Request strategy applied (optional)
:type request_strategy: :py:class:`RequestStrategyBase`
:param keep_tempfiles: Flag indicating how to treat temporary files
:type keep_tempfiles: :py:class:`KeepTempfiles`
:param str http_method: HTTP method used when issuing requests to
endpoints
:param float retry_budget_client: Per client retry-budget in percent.
The value defines the cut-off error ratio above requests to
datacenters (DC) are dropped.
:param proxy_netloc: Proxy netloc delegated to the routing service
in use
:type proxy_netloc: str or None
"""
self.mimetype = mimetype
self.content_type = (
'{}; {}'.format(self.mimetype, settings.CHARSET_TEXT)
if self.mimetype == settings.MIMETYPE_TEXT else self.mimetype)
self.query_params = query_params
self.stream_epochs = stream_epochs
# TODO(damb): Pass as ctor arg.
self._routing_service = current_app.config['ROUTING_SERVICE']
self._logger = logging.getLogger(kwargs.get('logger', self.LOGGER))
self._ctx = kwargs.get('context', Context(uuid.uuid4()))
if not self._ctx.locked:
self._ctx.acquire()
self.logger = ContextLoggerAdapter(self._logger, {'ctx': self._ctx})
self._keep_tempfiles = kwargs.get('keep_tempfiles', KeepTempfiles.NONE)
self._retry_budget_client = kwargs.get(
'retry_budget_client', self.DEFAULT_RETRY_BUDGET_CLIENT)
self._num_routes = 0
self._pool = None
self._results = []
self._sizes = []
self._default_endtime = datetime.datetime.utcnow()
self._nodata = int(self.query_params.get(
'nodata', settings.FDSN_DEFAULT_NO_CONTENT_ERROR_CODE))
# lookup resource configuration attributes
req_strategy = kwargs.get('request_strategy',
self.DEFAULT_REQUEST_STRATEGY)
if req_strategy not in self.ALLOWED_STRATEGIES:
raise ConfigurationError(
'Invalid strategy: {!r}'.format(req_strategy))
self._strategy = self._STRATEGY_MAP[req_strategy]
self._strategy = self._strategy(
context=self._ctx, default_endtime=self.DEFAULT_ENDTIME)
self._http_method = kwargs.get(
'request_method', settings.EIDA_FEDERATOR_DEFAULT_HTTP_METHOD)
self._num_threads = kwargs.get('num_threads', self.POOL_SIZE)
self._proxy_netloc = kwargs.get('proxy_netloc')
self._post = True