def create_event_writer(config, process_safe):
use_hec = scutil.is_true(config.get(tac.use_hec))
use_raw_hec = scutil.is_true(config.get(tac.use_raw_hec))
if use_hec or use_raw_hec:
# if use hec, leave each worker process/thread to create event writer
event_writer = None
else:
event_writer = ew.create_event_writer(config, process_safe)
event_writer.start()
return event_writer
def create_data_loader_mgr(config):
"""
create a data loader with default event_writer, job_scheudler
"""
import splunktalib.event_writer as ew
import splunktalib.schedule.scheduler as sched
writer = ew.create_event_writer(
config, scutil.is_true(config.get("use_multiprocess")))
scheduler = sched.Scheduler()
loader_mgr = DataLoaderManager(config, scheduler, writer)
return loader_mgr
def create_data_loader_mgr(config):
"""
create a data loader with default event_writer, job_scheudler
"""
import splunktalib.event_writer as ew
import splunktalib.schedule.scheduler as sched
writer = ew.create_event_writer(
config, scutil.is_true(config.get("use_multiprocess")))
scheduler = sched.Scheduler()
loader_mgr = DataLoaderManager(config, scheduler, writer)
return loader_mgr
def start(self):
if self._started:
return
self._started = True
self._timer_queue.start()
process_safe = self._use_multiprocess()
logger.info("Use multiprocessing=%s", process_safe)
event_writer = ew.create_event_writer(self._task_configs[0],
process_safe)
event_writer.start()
tear_down_q = self._create_tear_down_queue(process_safe)
loaders = []
for task in self._task_configs:
task[ggc.event_writer] = event_writer
loader = GoogleConcurrentDataLoader(task, tear_down_q,
process_safe)
loader.start()
loaders.append(loader)
logger.info("GoogleDataLoaderManager started")
_wait_for_tear_down(self._wakeup_queue, None)
logger.info("GoogleDataLoaderManager got stop signal")
for loader in loaders:
logger.info("Notify loader=%s", loader.name)
loader.stop()
event_writer.tear_down()
self._timer_queue.tear_down()
if self._mgr is not None:
self._mgr.shutdown()
logger.info("GoogleDataLoaderManager stopped")
def start(self):
if self._started:
return
self._started = True
self._timer_queue.start()
process_safe = self._use_multiprocess()
logger.info("Use multiprocessing=%s", process_safe)
event_writer = ew.create_event_writer(
self._task_configs[0], process_safe)
event_writer.start()
tear_down_q = self._create_tear_down_queue(process_safe)
loaders = []
for task in self._task_configs:
task[ggc.event_writer] = event_writer
loader = GoogleConcurrentDataLoader(
task, tear_down_q, process_safe)
loader.start()
loaders.append(loader)
logger.info("GoogleDataLoaderManager started")
_wait_for_tear_down(self._wakeup_queue, None)
logger.info("GoogleDataLoaderManager got stop signal")
for loader in loaders:
logger.info("Notify loader=%s", loader.name)
loader.stop()
event_writer.tear_down()
self._timer_queue.tear_down()
if self._mgr is not None:
self._mgr.shutdown()
logger.info("GoogleDataLoaderManager stopped")
