def _do_write_events(self):
event_queue = self._event_queue
write = sys.stdout.write
got_shutdown_signal = False
while 1:
try:
events = event_queue.get(timeout=3)
except Queue.Empty:
# We need drain the queue before shutdown
# timeout means empty for now
if got_shutdown_signal:
logger.info("ModinputEventWriter is going to exit...")
break
else:
continue
if events is not None:
if isinstance(events, (str, unicode)):
# for legacy interface
write(events)
else:
for event in events:
write(event.to_string())
sys.stdout.flush()
else:
logger.info("ModinputEventWriter got tear down signal")
got_shutdown_signal = True
def run(self, jobs):
if self._started:
return
self._started = True
self._event_writer.start()
self._executor.start()
self._timer_queue.start()
self._scheduler.start()
logger.info("TADataLoader started.")
def _enqueue_io_job(job):
job_props = job.get_props()
real_job = job_props["real_job"]
self.run_io_jobs((real_job, ))
for job in jobs:
j = sjob.Job(_enqueue_io_job, {"real_job": job},
job.get_interval())
self._scheduler.add_jobs((j, ))
self._wait_for_tear_down()
for job in jobs:
job.stop()
self._scheduler.tear_down()
self._timer_queue.tear_down()
self._executor.tear_down()
self._event_writer.tear_down()
logger.info("DataLoader stopped.")
def start(self):
if self._started:
return
self._started = True
self._event_writer.start()
logger.info("ModinputEventWriter started.")
def resize(self, new_size):
"""
Resize the pool size, spawn or destroy threads if necessary
"""
if new_size <= 0:
return
if self._lock.locked() or not self._started:
logger.info("Try to resize thread pool during the tear "
"down process, do nothing")
return
with self._lock:
self._remove_exited_threads_with_lock()
size = self._last_size
self._last_size = new_size
if new_size > size:
for _ in range(new_size - size):
thr = threading.Thread(target=self._run)
thr.daemon = self._daemon
thr.start()
self._thrs.append(thr)
elif new_size < size:
for _ in range(size - new_size):
self._work_queue.put(None)
logger.info("Finished ThreadPool resizing. New size=%d", new_size)
def _check_and_execute(self):
wakeup_queue = self._wakeup_queue
while 1:
(next_expired_time, expired_timers) = self._get_expired_timers()
for timer in expired_timers:
try:
timer()
except Exception:
logger.error(traceback.format_exc())
self._reset_timers(expired_timers)
# Calc sleep time
if next_expired_time:
now = time()
if now < next_expired_time:
sleep_time = next_expired_time - now
else:
sleep_time = 0.1
else:
sleep_time = 1
try:
wakeup = wakeup_queue.get(timeout=sleep_time)
if wakeup is None:
break
except Queue.Empty:
pass
logger.info("TimerQueue stopped.")
def _run(self):
"""
Threads callback func, run forever to handle jobs from the job queue
"""
work_queue = self._work_queue
count_lock = self._count_lock
while 1:
logger.debug("Going to get job")
func = work_queue.get()
if func is None:
break
if not self._started:
break
logger.debug("Going to exec job")
with count_lock:
self._occupied_threads += 1
try:
func()
except Exception:
logger.error(traceback.format_exc())
with count_lock:
self._occupied_threads -= 1
logger.debug("Done with exec job")
logger.info("Thread work_queue_size=%d", work_queue.qsize())
logger.debug("Worker thread %s stopped.",
threading.current_thread().getName())
def tear_down(self):
if not self._started:
return
self._started = False
self._event_queue.put(None)
self._event_writer.join()
logger.info("ModinputEventWriter stopped.")
def start(self):
"""
Start the timer queue to make it start function
"""
if self._started:
return
self._started = True
self._thr.start()
logger.info("TimerQueue started.")
def tear_down(self):
"""
Stop the schduler which will stop the internal thread for scheduling
jobs.
"""
if not self._started:
logger.info("Scheduler already tear down.")
return
self._wakeup_q.put(True)
def _wait_for_tear_down(self):
wakeup_q = self._wakeup_queue
while 1:
try:
go_exit = wakeup_q.get(timeout=1)
except queue.Empty:
pass
else:
if go_exit:
logger.info("DataLoader got stop signal")
self._stopped = True
break
def _wait_for_tear_down(self):
wakeup_q = self._wakeup_queue
while 1:
try:
go_exit = wakeup_q.get(timeout=1)
except Queue.Empty:
go_exit = self._orphan_checker.is_orphan()
if go_exit:
logger.info("DataLoaderManager got stop signal")
self._started = False
break
def start(self):
"""
Start the schduler which will start the internal thread for scheduling
jobs. Please do tear_down when doing cleanup
"""
if self._started:
logger.info("Scheduler already started.")
return
self._started = True
self._thr.start()
def _do_jobs(self):
while 1:
(sleep_time, jobs) = self.get_ready_jobs()
self._do_execution(jobs)
try:
done = self._wakeup_q.get(timeout=sleep_time)
except queue.Empty:
pass
else:
if done:
break
self._started = False
logger.info("Scheduler exited.")
def tear_down(self):
"""
Tear down the pool
"""
if self._stopped:
logger.info("ProcessPool has already stopped.")
return
self._stopped = True
self._pool.close()
self._pool.join()
logger.info("ProcessPool stopped.")
def apply(self, func, args=(), kwargs={}):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:return whatever the func returns
"""
if self._stopped:
logger.info("ProcessPool has already stopped.")
return None
return self._pool.apply(func, args, kwargs)
def enqueue_funcs(self, funcs, block=True):
"""
run jobs in a fire and forget way, no result will be handled
over to clients
:param funcs: tuple/list-like or generator like object, func shall be
callable
"""
if not self._started:
logger.info("ThreadPool has already stopped.")
return
for func in funcs:
self._work_queue.put(func, block)
def apply_async(self, func, args=(), kwargs={}, callback=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:callback: when func is done without exception, call this callack
:return whatever the func returns
"""
if self._stopped:
logger.info("ProcessPool has already stopped.")
return None
return self._pool.apply_async(func, args, kwargs, callback)
def remove_timer(self, timer):
"""
Remove timer from the queue.
"""
with self._lock:
try:
self._timers.remove(timer)
except ValueError:
logger.info(
"Timer=%s is not in queue, move it to cancelling "
"list", timer.ident())
else:
self._cancelling_timers[timer.ident()] = timer
def apply(self, func, args=(), kwargs=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:return whatever the func returns
"""
if not self._started:
logger.info("ThreadPool has already stopped.")
return None
res = self.apply_async(func, args, kwargs)
return res.get()
def apply_async(self, func, args=(), kwargs=None, callback=None):
"""
:param func: callable
:param args: free params
:param kwargs: named params
:callback: when func is done and without exception, call the callback
:return AsyncResult, clients can poll or wait the result through it
"""
if not self._started:
logger.info("ThreadPool has already stopped.")
return None
res = AsyncResult(func, args, kwargs, callback)
self._work_queue.put(res)
return res
def start(self):
"""
Start threads in the pool
"""
with self._lock:
if self._started:
return
self._started = True
for thr in self._thrs:
thr.daemon = self._daemon
thr.start()
self._admin_thr.start()
logger.info("ThreadPool started.")
def _do_admin(self):
admin_q = self._admin_queue
resize_win = self._resize_window
while 1:
try:
wakup = admin_q.get(timeout=resize_win + 1)
except queue.Empty:
self._do_resize_according_to_loads()
continue
if wakup is None:
break
else:
self._do_resize_according_to_loads()
logger.info("ThreadPool admin thread=%s stopped.",
threading.current_thread().getName())
def update_or_create_hec(config):
"""
:param config:
{
"server_uri": xxx,
"session_key": xxx,
"hec_name": xxx,
"hec_port": xxx,
"use_hec": 0/1,
"use_raw_hec": 0/1,
}
"""
use_hec = utils.is_true(config.get("use_hec"))
use_raw_hec = utils.is_true(config.get("use_raw_hec"))
if not use_hec and not use_raw_hec:
return
hec = HECConfig(config["server_uri"], config["session_key"])
hec_input = hec.get_http_input(config["hec_name"])
port = config.get("hec_port", 8088)
if not hec_input:
logger.info("Create HEC data input")
hec_settings = {
"enableSSL": 1,
"port": port,
"output_mode": "json",
"disabled": 0,
}
hec.update_settings(hec_settings)
input_settings = {
"name": config["hec_name"],
}
hec.create_http_input(input_settings)
hec_input = hec.get_http_input(config["hec_name"])
hostname, _ = utils.extract_hostname_port(config["server_uri"])
hec_uri = "https://{hostname}:{port}".format(hostname=hostname, port=port)
if hec_input:
hec_input[0]["hec_server_uri"] = hec_uri
return hec_input[0]
else:
raise Exception("Failed to get HTTP input configuration")
def check_changes(self):
logger.debug("Checking files=%s", self._files)
file_mtimes = self.file_mtimes
changed_files = []
for f, last_mtime in file_mtimes.items():
try:
current_mtime = op.getmtime(f)
if current_mtime != last_mtime:
file_mtimes[f] = current_mtime
changed_files.append(f)
logger.info("Detect %s has changed, last=%s, current=%s",
f, last_mtime, current_mtime)
except OSError:
pass
if changed_files:
if self._callback:
self._callback(changed_files)
return True
return False
def get_ready_jobs(self):
"""
@return: a 2 element tuple. The first element is the next ready
duration. The second element is ready jobs list
"""
now = time()
ready_jobs = []
sleep_time = 1
with self._lock:
job_set = self._jobs
total_jobs = len(job_set)
for job in job_set:
if job.get_expiration() <= now:
ready_jobs.append(job)
if ready_jobs:
del job_set[:len(ready_jobs)]
for job in ready_jobs:
if job.get_interval() != 0 and not job.stopped():
# repeated job, calculate next due time and enqueue
job.update_expiration()
job_set.add(job)
if job_set:
sleep_time = job_set[0].get_expiration() - now
if sleep_time < 0:
logger.warn("Scheduler satuation, sleep_time=%s",
sleep_time)
sleep_time = 0.1
if ready_jobs:
logger.info(
"Get %d ready jobs, next duration is %f, "
"and there are %s jobs scheduling", len(ready_jobs),
sleep_time, total_jobs)
ready_jobs.sort(key=lambda job: job.get("priority", 0), reverse=True)
return (sleep_time, ready_jobs)
def _do_resize_according_to_loads(self):
if (self._last_resize_time
and time() - self._last_resize_time < self._resize_window):
return
thr_size = self._last_size
free_thrs = thr_size - self._occupied_threads
work_size = self._work_queue.qsize()
logger.info("current_thr_size=%s, free_thrs=%s, work_size=%s",
thr_size, free_thrs, work_size)
if work_size and work_size > free_thrs:
if thr_size < self._max_size:
thr_size = min(thr_size * 2, self._max_size)
self.resize(thr_size)
elif free_thrs > 0:
if work_size // free_thrs < self._low_watermark and free_thrs >= 2:
thr_size = thr_size - free_thrs // 2
if thr_size > self._min_size:
self.resize(thr_size)
self._last_resize_time = time()
def tear_down(self):
"""
Tear down thread pool
"""
with self._lock:
if not self._started:
return
self._started = False
for thr in self._thrs:
self._work_queue.put(None, block=False)
self._admin_queue.put(None)
if not self._daemon:
logger.info("Wait for threads to stop.")
for thr in self._thrs:
thr.join()
self._admin_thr.join()
logger.info("ThreadPool stopped.")
def tear_down(self):
self._wakeup_queue.put(True)
logger.info("DataLoader is going to stop.")