def add_job(self, interval: int, job_type: JobTypeEnum, *args, **kwargs):
job = JOB_FACTORY(job_type)
j = job(interval=interval, time_unit=self.time_unit, *args, **kwargs)
with self._jobs_lock:
self._list_jobs.append(j)
Logger.info("Add a new Job: {}, interval: {} {}".format(
j.job_id, interval, self.time_unit.value))
return j
def _run_pending(self):
"""
Run pending jobs that should run
- Should run:: current time > Job's scheduled time
- Put each job in job queue of thread pool
- Checks that each Job can continue to the next run
"""
with self._jobs_lock:
list_jobs_to_iterate = self._list_jobs
Logger.info(str(list_jobs_to_iterate))
list_jobs_to_run = []
list_remaining_jobs = []
for job in list_jobs_to_iterate:
if job.should_run:
list_jobs_to_run.append(job)
else:
list_remaining_jobs.append(job)
with self._jobs_lock:
self._list_jobs = list_remaining_jobs
list_jobs_to_run = [
job for job in list_jobs_to_iterate if job.should_run
]
list_jobs_to_run = sorted(list_jobs_to_run)
list_threads = [
Thread(target=self._thread_pool.run,
kwargs={"func": job.run_job},
daemon=True) for job in list_jobs_to_run
]
for job_thread in list_threads:
job_thread.start()
for job_thread in list_threads:
job_thread.join()
for job in list_remaining_jobs:
if not job.can_continue():
job.cleanup()
for job in list_jobs_to_run:
if job.can_continue():
with self._jobs_lock:
self._list_jobs.append(job)
else:
job.cleanup()
def abort(self):
"""
Abort Job
- Only one thread will do it
- All other threads will return
- Simply sets status to TERMINATED
- Scheduler should understand that it shouldn't reschedule it
"""
with self._abort_lock:
if self._mark_for_abortion.is_set():
# All other threads will avoid cancellation
Logger.info("Job already cancelled: {}".format(self.job_id))
return
# Only one thread will cancel the job
if self.status in [JobStatusEnum.TERMINATED, JobStatusEnum.FAILED]:
Logger.warning("Job is already cancelled")
return
if self.status in [JobStatusEnum.RUNNING]:
Logger.warning(
"Job is already running - cannot abort it until finished")
return
# Only when job can be cancelled, we will raise a flag
self._mark_for_abortion.set()
Logger.info("Abort a Job: {}".format(self.job_id))
self.status = JobStatusEnum.TERMINATED
def run(self):
"""
Runs forever until joined
:return:
"""
self._thread_pool.start()
Logger.info("Scheduler Start")
while self._keep_running.is_set():
self._pre_run()
self._run_pending()
time.sleep(1)
self._post_run()
Logger.info("Scheduler Stopped")
def stop(self):
"""
Stop Pool
- Wait for all worker threads to finish
- Clears thread pool
:return:
"""
for worker in self._list_workers:
# 1 to avoid stuck - Workers are zombie threads
try:
worker.join(timeout=1)
except RuntimeError:
# Worker wasn't started - ignore
pass
Logger.info("Thread Pool stopped")
self._list_workers.clear()
def run(self):
"""
Worker function
- Runs until _keep_running event is down
- Takes a job from the job queue: Blocks for _TIMEOUT seconds
- Runs the function
- Prints
"""
Logger.info("Worker {} .. Start".format(self.name))
while self._keep_running.is_set():
try:
func, args, kwargs = self._task_queue.get(
block=True, timeout=ThreadPool.Worker._TIMEOUT)
try:
func(*args, **kwargs)
except Exception as e:
Logger.warning(e)
finally:
self._task_queue.task_done()
except Empty:
# DO nothing when found no job to do
# The wait is part of get job from queue
pass
Logger.info("Worker {} .. Done".format(self.name))
def run_job(self):
if self._mark_for_abortion.is_set():
return self.abort()
self._iterations += 1
self.status = JobStatusEnum.RUNNING
self._last_run_started = datetime.now()
try:
Logger.info("Running job: {}, #{}".format(self, self._iterations))
rc = self._job_func()
self._last_run_ended = datetime.now()
self.status = JobStatusEnum.SUCCEEDED
self._schedule_next_run()
except Exception as e:
self.status = JobStatusEnum.FAILED
Logger.warning(str(e))
Logger.warning(traceback.format_exc())
rc = str(e)
return rc
def func_to_run(self, contents):
Logger.info("Running function")
with open(TestThreadPool.ABS_FILE_PATH, "w+") as file_fd:
file_fd.write(contents)
def _do_job(self):
Logger.info("Running: {} [{}]".format(self._job_type, self.job_id))
self._last_result = None
def cleanup(self):
"""
Cleanup function that can be overriden
"""
Logger.info("Cleanup: {} [{}]".format(self._job_type, self.job_id))