def __init__(self, tile_service_factory, sc=None, **kwargs):
import inspect
NexusCalcHandler.__init__(self,
tile_service_factory=tile_service_factory,
**kwargs)
self.spark_job_stack = []
self._sc = sc
# max_concurrent_jobs = algorithm_config.getint("spark", "maxconcurrentjobs") if algorithm_config.has_section(
# "spark") and algorithm_config.has_option("spark", "maxconcurrentjobs") else 10
max_concurrent_jobs = 10
self.spark_job_stack = list(
["Job %s" % x for x in range(1, max_concurrent_jobs + 1)])
self.log = logging.getLogger(__name__)
def with_spark_job_context(calc_func):
from functools import wraps
@wraps(calc_func)
def wrapped(*args, **kwargs1):
try:
with NexusCalcSparkHandler.SparkJobContext(
self.spark_job_stack) as job_context:
# TODO Pool and Job are forced to a 1-to-1 relationship
calc_func.__self__._sc.setLocalProperty(
"spark.scheduler.pool", job_context.job_name)
calc_func.__self__._sc.setJobGroup(
job_context.job_name, "a spark job")
return calc_func(*args, **kwargs1)
except NexusCalcSparkHandler.SparkJobContext.MaxConcurrentJobsReached:
raise NexusProcessingException(
code=503,
reason=
"Max concurrent requests reached. Please try again later."
)
return wrapped
for member in inspect.getmembers(self, predicate=inspect.ismethod):
if member[0] == "calc":
setattr(self, member[0], with_spark_job_context(member[1]))
def __init__(self):
BaseHandler.__init__(self)
def __init__(self, tile_service_factory):
NexusCalcHandler.__init__(self, tile_service_factory)
def __init__(self, tile_service_factory):
BaseHandler.__init__(self, tile_service_factory)