class ExecutorService(object):
DEFAULT_VALUES = {
EXECUTOR_NAME: lambda: SPLUNK_EXECUTOR_SERVICE,
EXECUTOR_NUM_PROCS: lambda: 4,
EXECUTOR_POLL_TIME: lambda: 10,
EXECUTOR_POLL_TASK: lambda: None,
EXECUTOR_POLL_ARGS: lambda: list(),
EXECUTOR_POLL_KARGS: lambda: dict(),
EXECUTOR_START_POLLING_WITH_SVC: lambda: False,
EXECUTOR_SERVICE_START: lambda: False,
EXECUTOR_MAX_ITERATIONS: lambda: 0,
}
def __init__(self, **kargs):
self.pool = None
self.accepting_tasks = False
self.polling = False
self.polling_timer_thread = None
self.iterations = 0
for k, v in kargs.items():
if k in self.DEFAULT_VALUES and v is not None:
setattr(self, k, v)
elif k in self.DEFAULT_VALUES:
fn = self.DEFAULT_VALUES[k]
setattr(self, k, fn())
for k, fn in self.DEFAULT_VALUES.items():
if k not in kargs:
setattr(self, k, fn())
self.logger = Logger("executor-service-%s" % str(self.get_name()))
if self.get_service_start():
self.start()
def get_num_procs(self) -> int:
return getattr(self, EXECUTOR_NUM_PROCS)
def set_num_procs(self, num_procs: int):
return setattr(self, EXECUTOR_NUM_PROCS, num_procs)
def get_max_iterations(self) -> int:
return getattr(self, EXECUTOR_MAX_ITERATIONS)
def set_max_iterations(self, max_iterations: int):
return setattr(self, EXECUTOR_MAX_ITERATIONS, max_iterations)
def get_name(self) -> str:
return getattr(self, EXECUTOR_NAME)
def set_name(self, name: str):
return setattr(self, EXECUTOR_NAME, name)
def get_poll_args(self) -> list:
return getattr(self, EXECUTOR_POLL_ARGS)
def set_poll_args(self, poll_args: list):
return setattr(self, EXECUTOR_NAME, poll_args)
def get_poll_kargs(self) -> dict:
return getattr(self, EXECUTOR_POLL_KARGS)
def set_poll_kargs(self, poll_kargs: dict):
return setattr(self, EXECUTOR_NAME, poll_kargs)
def get_poll_time(self) -> int:
return getattr(self, EXECUTOR_POLL_TIME)
def set_poll_time(self, poll_time: int):
return setattr(self, EXECUTOR_POLL_TIME, poll_time)
def get_poll_task(self) -> callable:
return getattr(self, EXECUTOR_POLL_TASK)
def set_poll_task(self, poll_task: callable):
return setattr(self, EXECUTOR_POLL_TASK, poll_task)
def get_start_polling_with_service(self) -> bool:
return getattr(self, EXECUTOR_START_POLLING_WITH_SVC)
def set_start_polling_with_service(self, start_polling: bool):
setattr(self, EXECUTOR_START_POLLING_WITH_SVC, start_polling)
def get_service_start(self) -> bool:
return getattr(self, EXECUTOR_SERVICE_START)
@classmethod
def from_config(cls, name=None, **kwargs):
executor_cdict = None
if name is None:
executor_service_cdict = Config.get_value(EXECUTOR_SERVICE_BLOCK)
if executor_service_cdict is None:
raise Exception(
"No valid executor service configurations found")
if len(executor_service_cdict):
executor_cdict = list(executor_service_cdict.values())[0]
else:
executor_cdict = Config.get_executor_service(name)
if executor_cdict is None or len(executor_cdict) == 0:
raise Exception("No valid executor service configurations found")
skwargs = {}
for k, v in cls.DEFAULT_VALUES.items():
if k in kwargs:
skwargs[k] = kwargs[k]
elif k in executor_cdict:
skwargs[k] = executor_cdict.get(k)
else:
skwargs[k] = v()
return cls(**skwargs)
def poll_task(self):
'''
Execute the supplied task with keywords and arguments,
and then upon performing the poll_task take the results and submit
them as a job to the ExecutorService
The results returned from the supplied poll_task should be a list of dictionaries containing
the following keys and typed data:
{
"executor_job_func": callable,
"executor_job_callback": callable,
"executor_job_error_callback": callable,
"executor_job_args": list,
"executor_job_kargs": dict,
}
returns a Timer Thread (None, current, or newly scheduled) and whether or not a new Timer Thread was scheduled
:return: Timer, bool
'''
self.iterations += 1
self.logger.action('executor-poll_task', None,
'checking and preparing poll task')
poll_task = self.get_poll_task()
if not self.polling:
return self.polling_timer_thread, False
elif poll_task is None:
return self.polling_timer_thread, False
self.logger.action('executor-poll_task', None, 'executing poll_task')
args = self.get_poll_args()
kargs = self.get_poll_kargs()
args = [] if args is None else args
kargs = {} if kargs is None else kargs
results = poll_task(*args, **kargs)
self.logger.action(
'executor-poll_task', None,
'%s found %d results' % (poll_task.__name__, len(results)))
for result in results:
func: callable = result.get(EXECUTOR_JOB_FUNC, None)
callback: callable = result.get(EXECUTOR_JOB_CALLBACK, None)
error_callback: callable = result.get(EXECUTOR_JOB_ERROR_CALLBACK,
None)
jargs: list = result.get(EXECUTOR_JOB_ARGS, list())
jkargs: dict = result.get(EXECUTOR_JOB_KARGS, dict())
self.submit_job(func,
jargs,
jkargs,
callback=callback,
error_callback=error_callback)
self.polling_timer_thread = None
if self.polling and not self.is_iterations_completed():
return self.schedule_next_polling_timer()
return self.polling_timer_thread, False
def is_iterations_completed(self):
'''
Check to see if the execution service has completed the number of required iterations
:return: bool if we exceeded the number of poll iterations and max number of iterations is not zero
'''
mi = self.get_max_iterations()
if mi == 0 or self.iterations < mi:
return False
self.polling = False
return True
def schedule_next_polling_timer(self) -> (Timer, bool):
'''
Schedule a timer thread to execute a Executor.poll_task after Executor.get_poll_time() secconds
if Executor.polling is False, polling thread is not rescheduled
:return: Timer, bool Timer is None if not polling or previous thread
'''
if not self.polling:
return None, False
if not self.cancel_polling_timer():
return self.polling_timer_thread, False
poll_time = float(self.get_poll_time())
x = 'scheduling next task, current poll iteration: %.01fs' % poll_time
self.logger.action('executor-schedule_next_polling_timer', None, x)
self.polling_timer_thread = Timer(poll_time, self.poll_task)
self.polling_timer_thread.start()
return self.polling_timer_thread, True
def start_polling_timer(self):
'''
Start the continuous iterations of timer thread to execute a Executor.poll_task after
Executor.get_poll_time() secconds
if Executor.polling is False, polling thread is not rescheduled
:return: Timer, bool Timer is None if not polling or previous thread
:return:
'''
if self.polling:
return self.polling
poll_time = self.get_poll_time()
x = 'starting the polling timer: %.01fs' % poll_time
self.logger.action('executor-schedule_next_polling_timer', None, x)
self.iterations = 0
self.polling = True
self.schedule_next_polling_timer()
def cancel_polling_timer(self):
if self.polling_timer_thread is None or \
not isinstance(self.polling_timer_thread, Timer):
self.polling_timer_thread = None
return True
self.polling_timer_thread.cancel()
return not self.polling_timer_thread.is_alive() and \
self.polling_timer_thread.finished.is_set()
def stop_polling_timer(self):
self.polling = False
return self.cancel_polling_timer()
def start_service(self, with_polling=False):
return self.start(with_polling=with_polling)
def stop_service(self):
return self.stop()
def start(self, with_polling=False):
'''
start the executor process pool
:return:
'''
if self.pool is not None:
self.stop()
if with_polling or self.get_start_polling_with_service():
self.start_polling_timer()
self.logger.action('executor-start', None, 'starting the service')
self.accepting_tasks = True
self.pool = Pool(processes=self.get_num_procs())
self.logger.action('executor-start', None,
'starting the service completed')
def stop(self, join=True):
'''
stop the process pool. if join is true, join all processes first, otherwise terminate
:return:
'''
self.logger.action('executor-stop', None, 'stopping the service')
self.accepting_tasks = False
self.polling = False
self.stop_polling_timer()
if join:
self.join_terminate()
else:
self.terminate()
self.pool = None
def submit_job(self,
func: callable,
args: list,
kargs: dict,
callback: callable = None,
error_callback: callable = None) -> ApplyResult:
'''
:param func: callable, function to call
:param callback: callable to handle completion
:param error_callback: callable to handle errors
:param args: tuple of arguments for function
:param kwargs: keyword arguments for function
:return: ApplyResult, if callback not specified then the ApplyResult can be used to get the results if
the task can not be run, None is returned
'''
# callback = callback if self.callback is None else callback
name = None if callback is None else callback.__name__
if not self.accepting_tasks:
self.logger.action(
'executor-submit_job', None,
'declined to accept a job, no longer accepting')
return None
try:
self.logger.action(
'executor-submit_job', None,
'accepting a job (%s) with %d args and %d kwargs with a callback: %s'
% (func.__name__, len(args), len(kargs), name))
return self.pool.apply_async(func,
tuple(args),
kargs,
callback=callback,
error_callback=error_callback)
except:
self.logger.action('executor-submit_job', None,
'failed to accept a job')
return None
def terminate(self):
'''
terminate the process pool
:return:
'''
self.stop_polling_timer()
if self.pool is None:
self.logger.action('executor-terminate', None,
'no pool being used')
return
self.logger.action('executor-terminate', None,
'terminating and closing pool down')
self.pool.close()
self.pool.terminate()
self.pool = None
def join_terminate(self):
'''
join and then terminate the process pool
:return:
'''
self.stop_polling_timer()
self.accepting_tasks = False
self.logger.action('executor-join_terminate', None,
'ending job acceptance and koining pool')
try:
self.pool.join()
except:
pass
self.terminate()
def is_running(self):
return self.polling
def is_alive(self):
if self.polling_timer_thread is not None and self.polling_timer_thread.is_alive(
):
return False
if self.pool is not None and self.pool.is_alive():
return False
return True
@classmethod
def build_executor_results(cls,
func: callable,
jargs: list,
jkargs: dict,
callback: callable = None,
error_callback: callable = None):
result_params = {}
result_params[EXECUTOR_JOB_FUNC] = func
result_params[EXECUTOR_JOB_CALLBACK] = callback
result_params[EXECUTOR_JOB_ERROR_CALLBACK] = error_callback
result_params[EXECUTOR_JOB_ARGS] = jargs
result_params[EXECUTOR_JOB_KARGS] = jkargs
return result_params
